From 8d74d3f1adab3e155f879b4b20dee03781fa1044 Mon Sep 17 00:00:00 2001 From: Steph Prince <40640337+stephprince@users.noreply.github.com> Date: Thu, 29 Aug 2024 15:50:31 -0700 Subject: [PATCH 01/13] separate out recording container class --- CMakeLists.txt | 1 + src/nwb/NWBFile.cpp | 27 +----------- src/nwb/NWBFile.hpp | 59 +++----------------------- src/nwb/NWBRecording.cpp | 31 +++----------- src/nwb/NWBRecording.hpp | 26 ++---------- src/nwb/RecordingContainers.cpp | 50 ++++++++++++++++++++++ src/nwb/RecordingContainers.hpp | 74 +++++++++++++++++++++++++++++++++ tests/testNWBFile.cpp | 9 ++-- tests/testNWBRecording.cpp | 8 ++-- 9 files changed, 151 insertions(+), 134 deletions(-) create mode 100644 src/nwb/RecordingContainers.cpp create mode 100644 src/nwb/RecordingContainers.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index bf84abc2..59be782f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -24,6 +24,7 @@ add_library( src/hdf5/HDF5IO.cpp src/nwb/NWBFile.cpp src/nwb/NWBRecording.cpp + src/nwb/RecordingContainers.cpp src/nwb/base/TimeSeries.cpp src/nwb/device/Device.cpp src/nwb/ecephys/ElectricalSeries.cpp diff --git a/src/nwb/NWBFile.cpp b/src/nwb/NWBFile.cpp index 132840ca..3c7b5961 100644 --- a/src/nwb/NWBFile.cpp +++ b/src/nwb/NWBFile.cpp @@ -45,7 +45,6 @@ Status NWBFile::initialize() Status NWBFile::finalize() { - recordingContainers.reset(); return io->close(); } @@ -93,7 +92,8 @@ Status NWBFile::createFileStructure() Status NWBFile::createElectricalSeries( std::vector recordingArrays, - const BaseDataType& dataType) + const BaseDataType& dataType, + RecordingContainers* recordingContainers) { if (!io->canModifyObjects()) { return Status::Failure; @@ -182,26 +182,3 @@ std::unique_ptr NWBFile::createRecordingData( return std::unique_ptr( io->createArrayDataSet(type, size, chunking, path)); } - -TimeSeries* NWBFile::getTimeSeries(const SizeType& timeseriesInd) -{ - if (timeseriesInd >= this->recordingContainers->containers.size()) { - return nullptr; - } else { - return this->recordingContainers->containers[timeseriesInd].get(); - } -} - -// Recording Container - -RecordingContainers::RecordingContainers(const std::string& name) - : name(name) -{ -} - -RecordingContainers::~RecordingContainers() {} - -void RecordingContainers::addData(std::unique_ptr data) -{ - this->containers.push_back(std::move(data)); -} diff --git a/src/nwb/NWBFile.hpp b/src/nwb/NWBFile.hpp index 08a22d41..33af59ea 100644 --- a/src/nwb/NWBFile.hpp +++ b/src/nwb/NWBFile.hpp @@ -10,6 +10,7 @@ #include "BaseIO.hpp" #include "Types.hpp" #include "nwb/base/TimeSeries.hpp" +#include "nwb/RecordingContainers.hpp" /*! * \namespace AQNWB::NWB @@ -18,8 +19,6 @@ namespace AQNWB::NWB { -class RecordingContainers; // declare here because gets used in NWBFile class - /** * @brief The NWBFile class provides an interface for setting up and managing * the NWB file. @@ -69,12 +68,14 @@ class NWBFile * @param recordingArrays vector of ChannelVector indicating the electrodes to * record from. A separate ElectricalSeries will be * created for each ChannelVector. + * @param recordingContainers The container to store the created TimeSeries. * @param dataType The data type of the elements in the data block. * @return Status The status of the object creation operation. */ Status createElectricalSeries( std::vector recordingArrays, - const BaseDataType& dataType = BaseDataType::I16); + const BaseDataType& dataType = BaseDataType::I16, + RecordingContainers* recordingContainers = nullptr); /** * @brief Starts the recording. @@ -86,11 +87,6 @@ class NWBFile */ void stopRecording(); - /** - * @brief Gets the TimeSeries object from the recordingContainers - * @param timeseriesInd The index of the timeseries dataset within the group. - */ - TimeSeries* getTimeSeries(const SizeType& timeseriesInd); protected: /** @@ -133,53 +129,8 @@ class NWBFile const std::array, N>& specVariables); - /** - * @brief Holds the Container (usually TimeSeries) objects that have been - * created in the nwb file for recording. - */ - std::unique_ptr recordingContainers = - std::make_unique("RecordingContainers"); - const std::string identifierText; std::shared_ptr io; }; -/** - * @brief The RecordingContainers class provides an interface for managing - * groups of TimeSeries acquired during a recording. - */ -class RecordingContainers -{ -public: - /** - * @brief Constructor for RecordingContainer class. - * @param name The name of the group of time series - */ - RecordingContainers(const std::string& name); - - /** - * @brief Deleted copy constructor to prevent construction-copying. - */ - RecordingContainers(const RecordingContainers&) = delete; - - /** - * @brief Deleted copy assignment operator to prevent copying. - */ - RecordingContainers& operator=(const RecordingContainers&) = delete; - - /** - * @brief Destructor for RecordingContainer class. - */ - ~RecordingContainers(); - - /** - * @brief Adds a TimeSeries object to the container. - * @param data The TimeSeries object to add. - */ - void addData(std::unique_ptr data); - - std::vector> containers; - std::string name; -}; - -} // namespace AQNWB::NWB +} // namespace AQNWB::NWB \ No newline at end of file diff --git a/src/nwb/NWBRecording.cpp b/src/nwb/NWBRecording.cpp index 1747471b..d91c5073 100644 --- a/src/nwb/NWBRecording.cpp +++ b/src/nwb/NWBRecording.cpp @@ -19,7 +19,8 @@ NWBRecording::~NWBRecording() Status NWBRecording::openFile(const std::string& filename, std::vector recordingArrays, - const std::string& IOType) + const std::string& IOType, + RecordingContainers* recordingContainers) { // close any existing files if (nwbfile != nullptr) { @@ -32,7 +33,9 @@ Status NWBRecording::openFile(const std::string& filename, nwbfile->initialize(); // create the datasets - nwbfile->createElectricalSeries(recordingArrays); + nwbfile->createElectricalSeries(recordingArrays, + BaseDataType::I16, + recordingContainers); // start the new recording return nwbfile->startRecording(); @@ -43,27 +46,3 @@ void NWBRecording::closeFile() nwbfile->stopRecording(); nwbfile->finalize(); } - -Status NWBRecording::writeTimeseriesData( - const std::string& containerName, - const SizeType& timeseriesInd, - const Channel& channel, - const std::vector& dataShape, - const std::vector& positionOffset, - const void* data, - const void* timestamps) -{ - TimeSeries* ts = nwbfile->getTimeSeries(timeseriesInd); - - if (ts == nullptr) - return Status::Failure; - - // write data and timestamps to datasets - if (channel.localIndex == 0) { - // write with timestamps if it's the first channel - return ts->writeData(dataShape, positionOffset, data, timestamps); - } else { - // write without timestamps if its another channel in the same timeseries - return ts->writeData(dataShape, positionOffset, data); - } -} diff --git a/src/nwb/NWBRecording.hpp b/src/nwb/NWBRecording.hpp index c24edd3b..cbbfeea7 100644 --- a/src/nwb/NWBRecording.hpp +++ b/src/nwb/NWBRecording.hpp @@ -1,6 +1,7 @@ #pragma once #include "Types.hpp" +#include "nwb/RecordingContainers.hpp" #include "nwb/NWBFile.hpp" namespace AQNWB::NWB @@ -41,7 +42,8 @@ class NWBRecording */ Status openFile(const std::string& filename, std::vector recordingArrays, - const std::string& IOType = "HDF5"); + const std::string& IOType = "HDF5", + RecordingContainers* recordingContainers = nullptr); /** * @brief Closes the file and performs necessary cleanup when recording @@ -49,28 +51,6 @@ class NWBRecording */ void closeFile(); - /** - * @brief Write timeseries to an NWB file. - * @param containerName The name of the timeseries group to write to. - * @param timeseriesInd The index of the timeseries dataset within the - * timeseries group. - * @param channel The channel index to use for writing timestamps. - * @param dataShape The size of the data block. - * @param positionOffset The position of the data block to write to. - * @param data A pointer to the data block. - * @param timestamps A pointer to the timestamps block. May be null if - * multidimensional TimeSeries and only need to write the timestamps once but - * write data multiple times. - * @return The status of the write operation. - */ - Status writeTimeseriesData(const std::string& containerName, - const SizeType& timeseriesInd, - const Channel& channel, - const std::vector& dataShape, - const std::vector& positionOffset, - const void* data, - const void* timestamps); - private: /** * @brief Pointer to the current NWB file. diff --git a/src/nwb/RecordingContainers.cpp b/src/nwb/RecordingContainers.cpp new file mode 100644 index 00000000..73c0f31a --- /dev/null +++ b/src/nwb/RecordingContainers.cpp @@ -0,0 +1,50 @@ + +#include "nwb/RecordingContainers.hpp" +#include "nwb/base/TimeSeries.hpp" + +using namespace AQNWB::NWB; +// Recording Container + +RecordingContainers::RecordingContainers() +{ +} + +RecordingContainers::~RecordingContainers() {} + +void RecordingContainers::addData(std::unique_ptr data) +{ + this->containers.push_back(std::move(data)); +} + +TimeSeries* RecordingContainers::getTimeSeries(const SizeType& timeseriesInd) +{ + if (timeseriesInd >= this->containers.size()) { + return nullptr; + } else { + return this->containers[timeseriesInd].get(); + } +} + +Status RecordingContainers::writeTimeseriesData( + const SizeType& timeseriesInd, + const Channel& channel, + const std::vector& dataShape, + const std::vector& positionOffset, + const void* data, + const void* timestamps) +{ + TimeSeries* ts = getTimeSeries(timeseriesInd); + + if (ts == nullptr) + return Status::Failure; + + // write data and timestamps to datasets + if (channel.localIndex == 0) { + // write with timestamps if it's the first channel + return ts->writeData(dataShape, positionOffset, data, timestamps); + } else { + // write without timestamps if its another channel in the same timeseries + return ts->writeData(dataShape, positionOffset, data); + } +} + diff --git a/src/nwb/RecordingContainers.hpp b/src/nwb/RecordingContainers.hpp new file mode 100644 index 00000000..87b036cb --- /dev/null +++ b/src/nwb/RecordingContainers.hpp @@ -0,0 +1,74 @@ +#pragma once + +#include "Channel.hpp" +#include "Types.hpp" +#include "nwb/base/TimeSeries.hpp" + +namespace AQNWB::NWB +{ + +/** + * @brief The RecordingContainers class provides an interface for managing + * and holding groups of TimeSeries acquired during a recording. + */ + +class RecordingContainers +{ +public: + /** + * @brief Constructor for RecordingContainer class. + */ + RecordingContainers(); + + /** + * @brief Deleted copy constructor to prevent construction-copying. + */ + RecordingContainers(const RecordingContainers&) = delete; + + /** + * @brief Deleted copy assignment operator to prevent copying. + */ + RecordingContainers& operator=(const RecordingContainers&) = delete; + + /** + * @brief Destructor for RecordingContainer class. + */ + ~RecordingContainers(); + + /** + * @brief Adds a TimeSeries object to the container. + * @param data The TimeSeries object to add. + */ + void addData(std::unique_ptr data); + + /** + * @brief Gets the TimeSeries object from the recordingContainers + * @param timeseriesInd The index of the timeseries dataset within the group. + */ + TimeSeries* getTimeSeries(const SizeType& timeseriesInd); + + /** + * @brief Write timeseries data to a recordingContainer dataset. + * @param timeseriesInd The index of the timeseries dataset within the + * timeseries group. + * @param channel The channel index to use for writing timestamps. + * @param dataShape The size of the data block. + * @param positionOffset The position of the data block to write to. + * @param data A pointer to the data block. + * @param timestamps A pointer to the timestamps block. May be null if + * multidimensional TimeSeries and only need to write the timestamps once but + * write data multiple times. + * @return The status of the write operation. + */ + Status writeTimeseriesData(const SizeType& timeseriesInd, + const Channel& channel, + const std::vector& dataShape, + const std::vector& positionOffset, + const void* data, + const void* timestamps); + + std::vector> containers; + std::string name; +}; + +} // namespace AQNWB::NWB diff --git a/tests/testNWBFile.cpp b/tests/testNWBFile.cpp index bae35f42..d89335f3 100644 --- a/tests/testNWBFile.cpp +++ b/tests/testNWBFile.cpp @@ -4,6 +4,7 @@ #include "Utils.hpp" #include "hdf5/HDF5IO.hpp" #include "nwb/NWBFile.hpp" +#include "nwb/RecordingContainers.hpp" #include "nwb/base/TimeSeries.hpp" #include "testUtils.hpp" @@ -31,8 +32,10 @@ TEST_CASE("createElectricalSeries", "[nwb]") // create Electrical Series std::vector mockArrays = getMockChannelArrays(1, 2); + std::unique_ptr recordingContainers = + std::make_unique(); Status resultCreate = - nwbfile.createElectricalSeries(mockArrays, BaseDataType::F32); + nwbfile.createElectricalSeries(mockArrays, BaseDataType::F32, recordingContainers.get()); REQUIRE(resultCreate == Status::Success); // start recording @@ -45,10 +48,10 @@ TEST_CASE("createElectricalSeries", "[nwb]") std::vector positionOffset = {0, 0}; std::vector dataShape = {mockData.size(), 0}; - NWB::TimeSeries* ts0 = nwbfile.getTimeSeries(0); + NWB::TimeSeries* ts0 = recordingContainers->getTimeSeries(0); ts0->writeData( dataShape, positionOffset, mockData.data(), mockTimestamps.data()); - NWB::TimeSeries* ts1 = nwbfile.getTimeSeries(1); + NWB::TimeSeries* ts1 = recordingContainers->getTimeSeries(1); ts1->writeData( dataShape, positionOffset, mockData.data(), mockTimestamps.data()); diff --git a/tests/testNWBRecording.cpp b/tests/testNWBRecording.cpp index 9b43d034..ea910f2f 100644 --- a/tests/testNWBRecording.cpp +++ b/tests/testNWBRecording.cpp @@ -9,6 +9,7 @@ #include "Utils.hpp" #include "hdf5/HDF5IO.hpp" #include "nwb/NWBRecording.hpp" +#include "nwb/RecordingContainers.hpp" #include "nwb/file/ElectrodeTable.hpp" #include "testUtils.hpp" @@ -34,10 +35,12 @@ TEST_CASE("writeContinuousData", "[recording]") std::vector> mockData = getMockData2D(numSamples, numChannels); std::vector mockTimestamps = getMockTimestamps(numSamples); + std::unique_ptr recordingContainers = + std::make_unique(); // open files NWB::NWBRecording nwbRecording; - nwbRecording.openFile(path, mockRecordingArrays); + nwbRecording.openFile(path, mockRecordingArrays, "HDF5", recordingContainers.get()); // run recording bool isRecording = true; @@ -62,8 +65,7 @@ TEST_CASE("writeContinuousData", "[recording]") std::unique_ptr intBuffer = transformToInt16( dataBuffer.size(), channel.getBitVolts(), dataBuffer.data()); - nwbRecording.writeTimeseriesData("ElectricalSeries", - i, + recordingContainers->writeTimeseriesData(i, channel, dataShape, positionOffset, From 133a3280761eecc49a51cd0a1a42f18462d3b508 Mon Sep 17 00:00:00 2001 From: Steph Prince <40640337+stephprince@users.noreply.github.com> Date: Thu, 29 Aug 2024 16:12:16 -0700 Subject: [PATCH 02/13] remove NWBRecording class and update workflow --- CMakeLists.txt | 1 - src/Utils.hpp | 4 +- src/nwb/NWBFile.cpp | 10 ---- src/nwb/NWBFile.hpp | 13 +--- src/nwb/NWBRecording.cpp | 48 --------------- src/nwb/NWBRecording.hpp | 60 ------------------- src/nwb/RecordingContainers.cpp | 6 +- tests/CMakeLists.txt | 2 +- tests/testNWBFile.cpp | 16 ++--- ...ecording.cpp => testRecordingWorkflow.cpp} | 51 +++++++++------- 10 files changed, 44 insertions(+), 167 deletions(-) delete mode 100644 src/nwb/NWBRecording.cpp delete mode 100644 src/nwb/NWBRecording.hpp rename tests/{testNWBRecording.cpp => testRecordingWorkflow.cpp} (79%) diff --git a/CMakeLists.txt b/CMakeLists.txt index 59be782f..e46e90f3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -23,7 +23,6 @@ add_library( src/Channel.cpp src/hdf5/HDF5IO.cpp src/nwb/NWBFile.cpp - src/nwb/NWBRecording.cpp src/nwb/RecordingContainers.cpp src/nwb/base/TimeSeries.cpp src/nwb/device/Device.cpp diff --git a/src/Utils.hpp b/src/Utils.hpp index b1a85813..f98d5b30 100644 --- a/src/Utils.hpp +++ b/src/Utils.hpp @@ -58,11 +58,11 @@ inline std::string getCurrentTime() * @brief Factory method to create an IO object. * @return A pointer to a BaseIO object */ -inline std::unique_ptr createIO(const std::string& type, +inline std::shared_ptr createIO(const std::string& type, const std::string& filename) { if (type == "HDF5") { - return std::make_unique(filename); + return std::make_shared(filename); } else { throw std::invalid_argument("Invalid IO type"); } diff --git a/src/nwb/NWBFile.cpp b/src/nwb/NWBFile.cpp index 3c7b5961..5d519944 100644 --- a/src/nwb/NWBFile.cpp +++ b/src/nwb/NWBFile.cpp @@ -145,16 +145,6 @@ Status NWBFile::createElectricalSeries( return Status::Success; } -Status NWBFile::startRecording() -{ - return io->startRecording(); -} - -void NWBFile::stopRecording() -{ - io->stopRecording(); -} - template void NWBFile::cacheSpecifications( const std::string& specPath, diff --git a/src/nwb/NWBFile.hpp b/src/nwb/NWBFile.hpp index 33af59ea..61ecca04 100644 --- a/src/nwb/NWBFile.hpp +++ b/src/nwb/NWBFile.hpp @@ -9,8 +9,8 @@ #include "BaseIO.hpp" #include "Types.hpp" -#include "nwb/base/TimeSeries.hpp" #include "nwb/RecordingContainers.hpp" +#include "nwb/base/TimeSeries.hpp" /*! * \namespace AQNWB::NWB @@ -77,17 +77,6 @@ class NWBFile const BaseDataType& dataType = BaseDataType::I16, RecordingContainers* recordingContainers = nullptr); - /** - * @brief Starts the recording. - */ - Status startRecording(); - - /** - * @brief Stops the recording. - */ - void stopRecording(); - - protected: /** * @brief Creates the default file structure. diff --git a/src/nwb/NWBRecording.cpp b/src/nwb/NWBRecording.cpp deleted file mode 100644 index d91c5073..00000000 --- a/src/nwb/NWBRecording.cpp +++ /dev/null @@ -1,48 +0,0 @@ - -#include "nwb/NWBRecording.hpp" - -#include "Channel.hpp" -#include "Utils.hpp" -#include "hdf5/HDF5IO.hpp" - -using namespace AQNWB::NWB; - -// NWBRecordingEngine -NWBRecording::NWBRecording() {} - -NWBRecording::~NWBRecording() -{ - if (nwbfile != nullptr) { - nwbfile->finalize(); - } -} - -Status NWBRecording::openFile(const std::string& filename, - std::vector recordingArrays, - const std::string& IOType, - RecordingContainers* recordingContainers) -{ - // close any existing files - if (nwbfile != nullptr) { - this->closeFile(); - } - - // initialize nwbfile object and create base structure - nwbfile = std::make_unique(generateUuid(), - createIO(IOType, filename)); - nwbfile->initialize(); - - // create the datasets - nwbfile->createElectricalSeries(recordingArrays, - BaseDataType::I16, - recordingContainers); - - // start the new recording - return nwbfile->startRecording(); -} - -void NWBRecording::closeFile() -{ - nwbfile->stopRecording(); - nwbfile->finalize(); -} diff --git a/src/nwb/NWBRecording.hpp b/src/nwb/NWBRecording.hpp deleted file mode 100644 index cbbfeea7..00000000 --- a/src/nwb/NWBRecording.hpp +++ /dev/null @@ -1,60 +0,0 @@ -#pragma once - -#include "Types.hpp" -#include "nwb/RecordingContainers.hpp" -#include "nwb/NWBFile.hpp" - -namespace AQNWB::NWB -{ -/** - * @brief The NWBRecording class manages the recording process - */ - -class NWBRecording -{ -public: - /** - * @brief Default constructor for NWBRecording. - */ - NWBRecording(); - - /** - * @brief Deleted copy constructor to prevent construction-copying. - */ - NWBRecording(const NWBRecording&) = delete; - - /** - * @brief Deleted copy assignment operator to prevent copying. - */ - NWBRecording& operator=(const NWBRecording&) = delete; - - /** - * @brief Destructor for NWBRecordingEngine. - */ - ~NWBRecording(); - - /** - * @brief Opens the file for recording. - * @param filename The name of the file to open. - * @param recordingArrays ChannelVector objects indicating the electrodes to - * use for ElectricalSeries recordings - * @param IOType Type of backend IO to use - */ - Status openFile(const std::string& filename, - std::vector recordingArrays, - const std::string& IOType = "HDF5", - RecordingContainers* recordingContainers = nullptr); - - /** - * @brief Closes the file and performs necessary cleanup when recording - * stops. - */ - void closeFile(); - -private: - /** - * @brief Pointer to the current NWB file. - */ - std::unique_ptr nwbfile; -}; -} // namespace AQNWB::NWB diff --git a/src/nwb/RecordingContainers.cpp b/src/nwb/RecordingContainers.cpp index 73c0f31a..a8d7c8e9 100644 --- a/src/nwb/RecordingContainers.cpp +++ b/src/nwb/RecordingContainers.cpp @@ -1,13 +1,12 @@ #include "nwb/RecordingContainers.hpp" + #include "nwb/base/TimeSeries.hpp" using namespace AQNWB::NWB; // Recording Container -RecordingContainers::RecordingContainers() -{ -} +RecordingContainers::RecordingContainers() {} RecordingContainers::~RecordingContainers() {} @@ -47,4 +46,3 @@ Status RecordingContainers::writeTimeseriesData( return ts->writeData(dataShape, positionOffset, data); } } - diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index b830774f..f2e3fce4 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -17,7 +17,7 @@ add_executable(aqnwb_test testFile.cpp testHDF5IO.cpp testNWBFile.cpp - testNWBRecording.cpp + testRecordingWorkflow.cpp examples/test_HDF5IO_examples.cpp examples/test_example.cpp ) diff --git a/tests/testNWBFile.cpp b/tests/testNWBFile.cpp index d89335f3..15179d11 100644 --- a/tests/testNWBFile.cpp +++ b/tests/testNWBFile.cpp @@ -26,20 +26,20 @@ TEST_CASE("createElectricalSeries", "[nwb]") std::string filename = getTestFilePath("createElectricalSeries.nwb"); // initialize nwbfile object and create base structure - NWB::NWBFile nwbfile(generateUuid(), - std::make_unique(filename)); + std::shared_ptr io = std::make_shared(filename); + NWB::NWBFile nwbfile(generateUuid(), io); nwbfile.initialize(); // create Electrical Series std::vector mockArrays = getMockChannelArrays(1, 2); std::unique_ptr recordingContainers = std::make_unique(); - Status resultCreate = - nwbfile.createElectricalSeries(mockArrays, BaseDataType::F32, recordingContainers.get()); + Status resultCreate = nwbfile.createElectricalSeries( + mockArrays, BaseDataType::F32, recordingContainers.get()); REQUIRE(resultCreate == Status::Success); // start recording - Status resultStart = nwbfile.startRecording(); + Status resultStart = io->startRecording(); REQUIRE(resultStart == Status::Success); // write timeseries data @@ -63,12 +63,12 @@ TEST_CASE("setCanModifyObjectsMode", "[nwb]") std::string filename = getTestFilePath("testCanModifyObjectsMode.nwb"); // initialize nwbfile object and create base structure with HDF5IO object - NWB::NWBFile nwbfile(generateUuid(), - std::make_unique(filename)); + std::shared_ptr io = std::make_shared(filename); + NWB::NWBFile nwbfile(generateUuid(), io); nwbfile.initialize(); // start recording - Status resultStart = nwbfile.startRecording(); + Status resultStart = io->startRecording(); REQUIRE(resultStart == Status::Success); // test that modifying the file structure after starting the recording fails diff --git a/tests/testNWBRecording.cpp b/tests/testRecordingWorkflow.cpp similarity index 79% rename from tests/testNWBRecording.cpp rename to tests/testRecordingWorkflow.cpp index ea910f2f..cdcd9a9b 100644 --- a/tests/testNWBRecording.cpp +++ b/tests/testRecordingWorkflow.cpp @@ -8,7 +8,7 @@ #include "Types.hpp" #include "Utils.hpp" #include "hdf5/HDF5IO.hpp" -#include "nwb/NWBRecording.hpp" +#include "nwb/NWBFile.hpp" #include "nwb/RecordingContainers.hpp" #include "nwb/file/ElectrodeTable.hpp" #include "testUtils.hpp" @@ -19,10 +19,7 @@ TEST_CASE("writeContinuousData", "[recording]") { SECTION("test data and timestamps stream") { - // get file path and remove if exists - std::string path = getTestFilePath("testContinuousRecording1.nwb"); - - // setup mock data + // 0. setup mock data SizeType numChannels = 4; SizeType numSamples = 300; SizeType samplesRecorded = 0; @@ -35,14 +32,28 @@ TEST_CASE("writeContinuousData", "[recording]") std::vector> mockData = getMockData2D(numSamples, numChannels); std::vector mockTimestamps = getMockTimestamps(numSamples); + + // 1. create IO object + std::string path = getTestFilePath("testContinuousRecording1.nwb"); + std::shared_ptr io = createIO("HDF5", path); + + // 2. create RecordingContainers object std::unique_ptr recordingContainers = std::make_unique(); - // open files - NWB::NWBRecording nwbRecording; - nwbRecording.openFile(path, mockRecordingArrays, "HDF5", recordingContainers.get()); + // 3. create NWBFile object + std::unique_ptr nwbfile = + std::make_unique(generateUuid(), io); + nwbfile->initialize(); + + // 4. create datasets and add to recording containers + nwbfile->createElectricalSeries( + mockRecordingArrays, BaseDataType::I16, recordingContainers.get()); - // run recording + // 5. start the recording + io->startRecording(); + + // 6. write data during the recording bool isRecording = true; while (isRecording) { // write data to the file for each channel @@ -58,7 +69,7 @@ TEST_CASE("writeContinuousData", "[recording]") mockTimestamps.begin() + samplesRecorded + bufferSize, timestampsBuffer.begin()); - // write timseries data + // write timeseries data std::vector positionOffset = {samplesRecorded, channel.localIndex}; std::vector dataShape = {dataBuffer.size(), 1}; @@ -66,11 +77,11 @@ TEST_CASE("writeContinuousData", "[recording]") dataBuffer.size(), channel.getBitVolts(), dataBuffer.data()); recordingContainers->writeTimeseriesData(i, - channel, - dataShape, - positionOffset, - intBuffer.get(), - timestampsBuffer.data()); + channel, + dataShape, + positionOffset, + intBuffer.get(), + timestampsBuffer.data()); } } // check if recording is done @@ -79,7 +90,10 @@ TEST_CASE("writeContinuousData", "[recording]") isRecording = false; } } - nwbRecording.closeFile(); + + // 7. stop the recording and finalize the file + io->stopRecording(); + nwbfile->finalize(); // check contents of data std::string dataPath = "/acquisition/array0/data"; @@ -123,9 +137,4 @@ TEST_CASE("writeContinuousData", "[recording]") REQUIRE_THAT(timestampsOut, Catch::Matchers::Approx(mockTimestamps).margin(tolerance)); } - - SECTION("add a new recording number to the same file", "[recording]") - { - // TODO - } } From 7990b6aa7dcf8a7ba70a8981d8b50ed490189788 Mon Sep 17 00:00:00 2001 From: Steph Prince <40640337+stephprince@users.noreply.github.com> Date: Fri, 30 Aug 2024 12:25:21 -0500 Subject: [PATCH 03/13] add workflow to docs --- docs/pages/1_userdocs.dox | 1 + docs/pages/userdocs/workflow.dox | 83 +++++++++++++++++++ tests/CMakeLists.txt | 1 + tests/examples/testWorkflowExamples.cpp | 104 ++++++++++++++++++++++++ 4 files changed, 189 insertions(+) create mode 100644 docs/pages/userdocs/workflow.dox create mode 100644 tests/examples/testWorkflowExamples.cpp diff --git a/docs/pages/1_userdocs.dox b/docs/pages/1_userdocs.dox index 20ebced1..e66d58e2 100644 --- a/docs/pages/1_userdocs.dox +++ b/docs/pages/1_userdocs.dox @@ -6,4 +6,5 @@ * * - \subpage user_install_page * - \subpage hdf5io + * - \subpage workflow */ diff --git a/docs/pages/userdocs/workflow.dox b/docs/pages/userdocs/workflow.dox new file mode 100644 index 00000000..d370eca0 --- /dev/null +++ b/docs/pages/userdocs/workflow.dox @@ -0,0 +1,83 @@ +/** + * \page workflow AqNWB Workflow + * + * \tableofcontents + * + * \section recording_workflow Overview of a recording workflow + * + * For users wanting to integrate NWB with a particular data acquisition software, here + * we outline the steps for a single recording from file creation to saving. + * + * 1. Create the I/O object. + * 2. Create the `RecordingContainers` object. + * 3. Create the `NWBFile` object. + * 4. Create datasets and add to RecordingContainers. + * 5. Start the recording. + * 6. Write data. + * 7. Stop the recording and close the NWBFile. + * + * Below, we walk through these steps in more detail. + * + * + * \subsection create_io 1. Create the I/O object. + * + * First, create an I/O object. We provide a convenience method, `createIO` + * to create this object using one of our supported backends. For more fine-grained + * control of different backend parameters, you can create your own `std::shared_ptr` + * using any of the derived BaseIO classes. + * + * \snippet tests/examples/testWorkflowExamples.cpp example_workflow_io_snippet + * + * + * \subsection create_recording_container 2. Create the RecordingContainer object. + * + * Next, create a RecordingContainer object to manage the different datasets + * that you would like to write data to. + * + * \snippet tests/examples/testWorkflowExamples.cpp example_workflow_recording_containers_snippet + * + * + * \subsection create_nwbfile 3. Create the NWBFile + * + * Next, constructs the `NWBFile` object, using the I/O object as an input. + * Then, initialize the object to create the basic file structure of the NWBFile. + * + * \snippet tests/examples/testWorkflowExamples.cpp example_workflow_nwbfile_snippet + * + * + * \subsection create_datasets 4. Create datasets and add to RecordingContainers. + * + * Next, create the different data types (e.g. `ElectricalSeries`, other `TimeSeries`) + * that you would like to write data into. After creation, these objects are moved + * to the RecordingContainers object so that it can manage access and data writing + * during the recording process. + * + * \snippet tests/examples/testWorkflowExamples.cpp example_workflow_datasets_snippet + * + * + * \subsection start_recording 5. Start the recording. + * + * Then, start the recording process with a call to the I/O object. By default for + * HDF5 files, this will enable SWMR mode and no additional datasets or groups can be added + * to the file unless it is closed and reopened. + * + * \snippet tests/examples/testWorkflowExamples.cpp example_workflow_start_snippet + * + * + * \subsection write_data 6. Write data. + * + * During the recording process, use the RecordingContainers as an interface + * to access the various datasets and write blocks of data to the file. + * + * \snippet tests/examples/testWorkflowExamples.cpp example_workflow_write_snippet + * + * + * \subsection stop_recording 7. Stop the recording and finalize the file. + * + * When the recording process is finished, call `stopRecording` from the I/O object + * to flush any data and close the file. + * + * \snippet tests/examples/testWorkflowExamples.cpp example_workflow_stop_snippet + * + * + */ diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index f2e3fce4..1add110d 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -20,6 +20,7 @@ add_executable(aqnwb_test testRecordingWorkflow.cpp examples/test_HDF5IO_examples.cpp examples/test_example.cpp + examples/testWorkflowExamples.cpp ) # Ensure the aqnwb_test target can include headers from the current directory diff --git a/tests/examples/testWorkflowExamples.cpp b/tests/examples/testWorkflowExamples.cpp new file mode 100644 index 00000000..f2adc667 --- /dev/null +++ b/tests/examples/testWorkflowExamples.cpp @@ -0,0 +1,104 @@ + +#include + +#include "BaseIO.hpp" +#include "Channel.hpp" +#include "Types.hpp" +#include "Utils.hpp" +#include "hdf5/HDF5IO.hpp" +#include "nwb/NWBFile.hpp" +#include "nwb/RecordingContainers.hpp" +#include "nwb/file/ElectrodeTable.hpp" +#include "testUtils.hpp" + +using namespace AQNWB; + +TEST_CASE("workflowExamples") +{ + SECTION("write workflow") + { + // 0. setup mock data + SizeType numChannels = 4; + SizeType numSamples = 300; + SizeType samplesRecorded = 0; + SizeType bufferSize = numSamples / 10; + std::vector dataBuffer(bufferSize); + std::vector timestampsBuffer(bufferSize); + + std::vector mockRecordingArrays = + getMockChannelArrays(); + std::vector> mockData = + getMockData2D(numSamples, numChannels); + std::vector mockTimestamps = getMockTimestamps(numSamples); + + std::string path = getTestFilePath("exampleRecording.nwb"); + // [example_workflow_io_snippet] + std::shared_ptr io = createIO("HDF5", path); + // [example_workflow_io_snippet] + + // [example_workflow_recording_containers_snippet] + std::unique_ptr recordingContainers = + std::make_unique(); + // [example_workflow_recording_containers_snippet] + + // [example_workflow_nwbfile_snippet] + std::unique_ptr nwbfile = + std::make_unique(generateUuid(), io); + nwbfile->initialize(); + // [example_workflow_nwbfile_snippet] + + // [example_workflow_datasets_snippet] + nwbfile->createElectricalSeries( + mockRecordingArrays, BaseDataType::I16, recordingContainers.get()); + // [example_workflow_datasets_snippet] + + // [example_workflow_start_snippet] + io->startRecording(); + // [example_workflow_start_snippet] + + // write data during the recording + bool isRecording = true; + while (isRecording) { + // write data to the file for each channel + for (SizeType i = 0; i < mockRecordingArrays.size(); ++i) { + const auto& channelVector = mockRecordingArrays[i]; + for (const auto& channel : channelVector) { + // copy data into buffer + std::copy(mockData[channel.globalIndex].begin() + samplesRecorded, + mockData[channel.globalIndex].begin() + samplesRecorded + + bufferSize, + dataBuffer.begin()); + std::copy(mockTimestamps.begin() + samplesRecorded, + mockTimestamps.begin() + samplesRecorded + bufferSize, + timestampsBuffer.begin()); + + // write timeseries data + std::vector positionOffset = {samplesRecorded, + channel.localIndex}; + std::vector dataShape = {dataBuffer.size(), 1}; + std::unique_ptr intBuffer = transformToInt16( + dataBuffer.size(), channel.getBitVolts(), dataBuffer.data()); + + // [example_workflow_write_snippet] + recordingContainers->writeTimeseriesData(i, + channel, + dataShape, + positionOffset, + intBuffer.get(), + timestampsBuffer.data()); + // [example_workflow_write_snippet] + } + } + // check if recording is done + samplesRecorded += dataBuffer.size(); + if (samplesRecorded >= numSamples) { + isRecording = false; + } + } + + // [example_workflow_stop_snippet] + io->stopRecording(); + nwbfile->finalize(); + // [example_workflow_stop_snippet] + } +} From a0b1706526f11634165de762d76664b87364e1ea Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Fri, 30 Aug 2024 11:11:29 -0700 Subject: [PATCH 04/13] swap order of hdf5io and workflow --- docs/pages/1_userdocs.dox | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/pages/1_userdocs.dox b/docs/pages/1_userdocs.dox index e66d58e2..ec3784fb 100644 --- a/docs/pages/1_userdocs.dox +++ b/docs/pages/1_userdocs.dox @@ -5,6 +5,6 @@ * with a particular data acquisition software via AqNWB. * * - \subpage user_install_page - * - \subpage hdf5io * - \subpage workflow + * - \subpage hdf5io */ From ee5a4feb0f68e6c8e613477f14fed0f0a3f2d89d Mon Sep 17 00:00:00 2001 From: Steph Prince <40640337+stephprince@users.noreply.github.com> Date: Fri, 30 Aug 2024 15:38:27 -0500 Subject: [PATCH 05/13] Apply suggestions from code review Co-authored-by: Oliver Ruebel --- docs/pages/userdocs/workflow.dox | 41 +++++++++++++++++++------------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/docs/pages/userdocs/workflow.dox b/docs/pages/userdocs/workflow.dox index d370eca0..36f7501d 100644 --- a/docs/pages/userdocs/workflow.dox +++ b/docs/pages/userdocs/workflow.dox @@ -8,30 +8,31 @@ * For users wanting to integrate NWB with a particular data acquisition software, here * we outline the steps for a single recording from file creation to saving. * - * 1. Create the I/O object. - * 2. Create the `RecordingContainers` object. - * 3. Create the `NWBFile` object. - * 4. Create datasets and add to RecordingContainers. + * 1. Create the I/O object (e.g,. \ref AQNWB::HDF5::HDF5IO "HDF5IO") used for writing data to the file on disk. + * 2. Create the \ref AQNWB::NWB::RecordingContainers "RecordingContainers" object used for managing \ref AQNWB::NWB::Container "Container" objects for storing recordings. + * 3. Create the \ref AQNWB::NWB::NWBFile "NWBFile" object used for managing and creating NWB file contents. + * 4. Create the \ref AQNWB::NWB::Container "Container" objects (e.g., \ref AQNWB::NWB::ElectricalSeries "ElectricalSeries") used for recording and add them to the \ref AQNWB::NWB::RecordingContainers "RecordingContainers". * 5. Start the recording. * 6. Write data. - * 7. Stop the recording and close the NWBFile. + * 7. Stop the recording and close the \ref AQNWB::NWB::NWBFile "NWBFile". * * Below, we walk through these steps in more detail. * * * \subsection create_io 1. Create the I/O object. * - * First, create an I/O object. We provide a convenience method, `createIO` + * First, create an I/O object (e.g., \ref AQNWB::HDF5::HDF5IO "HDF5IO") used for writing data to the file. + * We provide a convenience method, \ref AQNWB::createIO "createIO" * to create this object using one of our supported backends. For more fine-grained * control of different backend parameters, you can create your own `std::shared_ptr` - * using any of the derived BaseIO classes. + * using any of the derived \ref AQNWB::BaseIO "BaseIO" classes. * * \snippet tests/examples/testWorkflowExamples.cpp example_workflow_io_snippet * * * \subsection create_recording_container 2. Create the RecordingContainer object. * - * Next, create a RecordingContainer object to manage the different datasets + * Next, create a \ref AQNWB::NWB::RecordingContainers "RecordingContainers" object to manage the different \ref AQNWB::NWB::Container "Container" objects with the datasets * that you would like to write data to. * * \snippet tests/examples/testWorkflowExamples.cpp example_workflow_recording_containers_snippet @@ -39,7 +40,7 @@ * * \subsection create_nwbfile 3. Create the NWBFile * - * Next, constructs the `NWBFile` object, using the I/O object as an input. + * Next, constructs the \ref AQNWB::NWB::NWBFile "NWBFile" object, using the I/O object as an input. * Then, initialize the object to create the basic file structure of the NWBFile. * * \snippet tests/examples/testWorkflowExamples.cpp example_workflow_nwbfile_snippet @@ -47,9 +48,9 @@ * * \subsection create_datasets 4. Create datasets and add to RecordingContainers. * - * Next, create the different data types (e.g. `ElectricalSeries`, other `TimeSeries`) - * that you would like to write data into. After creation, these objects are moved - * to the RecordingContainers object so that it can manage access and data writing + * Next, create the different data types (e.g. \ref AQNWB::NWB::ElectricalSeries "ElectricalSeries" or other AQNWB::NWB::TimeSeries "TimeSeries") + * that you would like to write data into. After creation, these objects are added + * to the \ref AQNWB::NWB::RecordingContainers "RecordingContainers" object so that it can manage access and data writing * during the recording process. * * \snippet tests/examples/testWorkflowExamples.cpp example_workflow_datasets_snippet @@ -57,17 +58,23 @@ * * \subsection start_recording 5. Start the recording. * - * Then, start the recording process with a call to the I/O object. By default for - * HDF5 files, this will enable SWMR mode and no additional datasets or groups can be added - * to the file unless it is closed and reopened. + * Then, start the recording process with a call to the ``startRecording`` function of the I/O object. + * + * \note + * When using \ref AQNWB::HDF5::HDF5IO "HDF5IO" for writing to HDF5, calling + * \ref AQNWB::HDF5::HDF5IO::startRecording "startRecording" will by default enable + * \ref hdf5io_swmr "SWMR mode" to ensure file integrity and support concurrent read. + * As a result, no additional datasets or groups can be added to the file once a recording + * has been started unless the file is is closed and reopened. * * \snippet tests/examples/testWorkflowExamples.cpp example_workflow_start_snippet * * * \subsection write_data 6. Write data. * - * During the recording process, use the RecordingContainers as an interface - * to access the various datasets and write blocks of data to the file. + * During the recording process, use the \ref AQNWB::NWB::RecordingContainers "RecordingContainers" as an interface + * to access the various \ref AQNWB::NWB::Container "Container" object and corresponding + * datasets and write blocks of data to the file. * * \snippet tests/examples/testWorkflowExamples.cpp example_workflow_write_snippet * From 1e5a5f29166843897b1a55bbea18a50c13340b39 Mon Sep 17 00:00:00 2001 From: Steph Prince <40640337+stephprince@users.noreply.github.com> Date: Fri, 30 Aug 2024 15:40:36 -0500 Subject: [PATCH 06/13] make recordingContainers apply to any container object --- src/nwb/RecordingContainers.cpp | 31 +++++++++++++++++++++------ src/nwb/RecordingContainers.hpp | 38 ++++++++++++++++++++++++--------- tests/testNWBFile.cpp | 6 ++++-- 3 files changed, 56 insertions(+), 19 deletions(-) diff --git a/src/nwb/RecordingContainers.cpp b/src/nwb/RecordingContainers.cpp index a8d7c8e9..a0a5bc5a 100644 --- a/src/nwb/RecordingContainers.cpp +++ b/src/nwb/RecordingContainers.cpp @@ -1,7 +1,8 @@ #include "nwb/RecordingContainers.hpp" -#include "nwb/base/TimeSeries.hpp" +#include "nwb/ecephys/ElectricalSeries.hpp" +#include "nwb/hdmf/base/Container.hpp" using namespace AQNWB::NWB; // Recording Container @@ -10,29 +11,29 @@ RecordingContainers::RecordingContainers() {} RecordingContainers::~RecordingContainers() {} -void RecordingContainers::addData(std::unique_ptr data) +void RecordingContainers::addData(std::unique_ptr data) { this->containers.push_back(std::move(data)); } -TimeSeries* RecordingContainers::getTimeSeries(const SizeType& timeseriesInd) +Container* RecordingContainers::getContainer(const SizeType& containerInd) { - if (timeseriesInd >= this->containers.size()) { + if (containerInd >= this->containers.size()) { return nullptr; } else { - return this->containers[timeseriesInd].get(); + return this->containers[containerInd].get(); } } Status RecordingContainers::writeTimeseriesData( - const SizeType& timeseriesInd, + const SizeType& containerInd, const Channel& channel, const std::vector& dataShape, const std::vector& positionOffset, const void* data, const void* timestamps) { - TimeSeries* ts = getTimeSeries(timeseriesInd); + TimeSeries* ts = dynamic_cast(getContainer(containerInd)); if (ts == nullptr) return Status::Failure; @@ -46,3 +47,19 @@ Status RecordingContainers::writeTimeseriesData( return ts->writeData(dataShape, positionOffset, data); } } + +Status RecordingContainers::writeElectricalSeriesData( + const SizeType& containerInd, + const Channel& channel, + const SizeType& numSamples, + const void* data, + const void* timestamps) +{ + ElectricalSeries* es = + dynamic_cast(getContainer(containerInd)); + + if (es == nullptr) + return Status::Failure; + + es->writeChannel(channel.localIndex, numSamples, data, timestamps); +} diff --git a/src/nwb/RecordingContainers.hpp b/src/nwb/RecordingContainers.hpp index 87b036cb..c2fe0b40 100644 --- a/src/nwb/RecordingContainers.hpp +++ b/src/nwb/RecordingContainers.hpp @@ -9,7 +9,7 @@ namespace AQNWB::NWB /** * @brief The RecordingContainers class provides an interface for managing - * and holding groups of TimeSeries acquired during a recording. + * and holding groups of Containers acquired during a recording. */ class RecordingContainers @@ -36,20 +36,20 @@ class RecordingContainers ~RecordingContainers(); /** - * @brief Adds a TimeSeries object to the container. - * @param data The TimeSeries object to add. + * @brief Adds a Container object to the container. + * @param data The Container object to add. */ - void addData(std::unique_ptr data); + void addData(std::unique_ptr data); /** - * @brief Gets the TimeSeries object from the recordingContainers - * @param timeseriesInd The index of the timeseries dataset within the group. + * @brief Gets the Container object from the recordingContainers + * @param containerInd The index of the container dataset within the group. */ - TimeSeries* getTimeSeries(const SizeType& timeseriesInd); + Container* getContainer(const SizeType& containerInd); /** * @brief Write timeseries data to a recordingContainer dataset. - * @param timeseriesInd The index of the timeseries dataset within the + * @param containerInd The index of the timeseries dataset within the * timeseries group. * @param channel The channel index to use for writing timestamps. * @param dataShape The size of the data block. @@ -60,14 +60,32 @@ class RecordingContainers * write data multiple times. * @return The status of the write operation. */ - Status writeTimeseriesData(const SizeType& timeseriesInd, + Status writeTimeseriesData(const SizeType& containerInd, const Channel& channel, const std::vector& dataShape, const std::vector& positionOffset, const void* data, const void* timestamps); - std::vector> containers; + /** + * @brief Write ElectricalSereis data to a recordingContainer dataset. + * @param containerInd The index of the electrical series dataset within the + * electrical series group. + * @param channel The channel index to use for writing timestamps. + * @param dataShape The size of the data block. + * @param data A pointer to the data block. + * @param timestamps A pointer to the timestamps block. May be null if + * multidimensional TimeSeries and only need to write the timestamps once but + * write data multiple times. + * @return The status of the write operation. + */ + Status writeElectricalSeriesData(const SizeType& containerInd, + const Channel& channel, + const SizeType& numSamples, + const void* data, + const void* timestamps); + + std::vector> containers; std::string name; }; diff --git a/tests/testNWBFile.cpp b/tests/testNWBFile.cpp index 15179d11..4a6f0d45 100644 --- a/tests/testNWBFile.cpp +++ b/tests/testNWBFile.cpp @@ -48,10 +48,12 @@ TEST_CASE("createElectricalSeries", "[nwb]") std::vector positionOffset = {0, 0}; std::vector dataShape = {mockData.size(), 0}; - NWB::TimeSeries* ts0 = recordingContainers->getTimeSeries(0); + NWB::TimeSeries* ts0 = + static_cast(recordingContainers->getContainer(0)); ts0->writeData( dataShape, positionOffset, mockData.data(), mockTimestamps.data()); - NWB::TimeSeries* ts1 = recordingContainers->getTimeSeries(1); + NWB::TimeSeries* ts1 = + static_cast(recordingContainers->getContainer(1)); ts1->writeData( dataShape, positionOffset, mockData.data(), mockTimestamps.data()); From 584fa4c946eb227cbc55bb2c61e7d453aba36ebd Mon Sep 17 00:00:00 2001 From: Steph Prince <40640337+stephprince@users.noreply.github.com> Date: Fri, 30 Aug 2024 15:55:33 -0500 Subject: [PATCH 07/13] add optional io flush call --- docs/pages/userdocs/workflow.dox | 3 ++- tests/examples/testWorkflowExamples.cpp | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/pages/userdocs/workflow.dox b/docs/pages/userdocs/workflow.dox index 36f7501d..98e4bdd1 100644 --- a/docs/pages/userdocs/workflow.dox +++ b/docs/pages/userdocs/workflow.dox @@ -74,7 +74,8 @@ * * During the recording process, use the \ref AQNWB::NWB::RecordingContainers "RecordingContainers" as an interface * to access the various \ref AQNWB::NWB::Container "Container" object and corresponding - * datasets and write blocks of data to the file. + * datasets and write blocks of data to the file. Calling `flush()` on the I/O object at anytime will ensure the + * data is moved to disk. * * \snippet tests/examples/testWorkflowExamples.cpp example_workflow_write_snippet * diff --git a/tests/examples/testWorkflowExamples.cpp b/tests/examples/testWorkflowExamples.cpp index f2adc667..759ec4da 100644 --- a/tests/examples/testWorkflowExamples.cpp +++ b/tests/examples/testWorkflowExamples.cpp @@ -86,6 +86,7 @@ TEST_CASE("workflowExamples") positionOffset, intBuffer.get(), timestampsBuffer.data()); + io->flush(); // [example_workflow_write_snippet] } } From c39efc11b5257faa599ebeb90beed43bbee62ece Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Fri, 30 Aug 2024 17:11:27 -0700 Subject: [PATCH 08/13] Fix line breaks in workflow.dox --- docs/pages/userdocs/workflow.dox | 34 +++++++++++++++++++------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/docs/pages/userdocs/workflow.dox b/docs/pages/userdocs/workflow.dox index 98e4bdd1..ad02aafa 100644 --- a/docs/pages/userdocs/workflow.dox +++ b/docs/pages/userdocs/workflow.dox @@ -8,10 +8,15 @@ * For users wanting to integrate NWB with a particular data acquisition software, here * we outline the steps for a single recording from file creation to saving. * - * 1. Create the I/O object (e.g,. \ref AQNWB::HDF5::HDF5IO "HDF5IO") used for writing data to the file on disk. - * 2. Create the \ref AQNWB::NWB::RecordingContainers "RecordingContainers" object used for managing \ref AQNWB::NWB::Container "Container" objects for storing recordings. - * 3. Create the \ref AQNWB::NWB::NWBFile "NWBFile" object used for managing and creating NWB file contents. - * 4. Create the \ref AQNWB::NWB::Container "Container" objects (e.g., \ref AQNWB::NWB::ElectricalSeries "ElectricalSeries") used for recording and add them to the \ref AQNWB::NWB::RecordingContainers "RecordingContainers". + * 1. Create the I/O object (e.g,. \ref AQNWB::HDF5::HDF5IO "HDF5IO") used for + * writing data to the file on disk. + * 2. Create the \ref AQNWB::NWB::RecordingContainers "RecordingContainers" object + * used for managing \ref AQNWB::NWB::Container "Container" objects for storing recordings. + * 3. Create the \ref AQNWB::NWB::NWBFile "NWBFile" object used for managing and creating NWB + * file contents. + * 4. Create the \ref AQNWB::NWB::Container "Container" objects (e.g., + * \ref AQNWB::NWB::ElectricalSeries "ElectricalSeries") used for recording and add them + * to the \ref AQNWB::NWB::RecordingContainers "RecordingContainers". * 5. Start the recording. * 6. Write data. * 7. Stop the recording and close the \ref AQNWB::NWB::NWBFile "NWBFile". @@ -32,8 +37,9 @@ * * \subsection create_recording_container 2. Create the RecordingContainer object. * - * Next, create a \ref AQNWB::NWB::RecordingContainers "RecordingContainers" object to manage the different \ref AQNWB::NWB::Container "Container" objects with the datasets - * that you would like to write data to. + * Next, create a \ref AQNWB::NWB::RecordingContainers "RecordingContainers" object to manage the + * different \ref AQNWB::NWB::Container "Container" objects with the datasets that you would + * like to write data to. * * \snippet tests/examples/testWorkflowExamples.cpp example_workflow_recording_containers_snippet * @@ -48,10 +54,10 @@ * * \subsection create_datasets 4. Create datasets and add to RecordingContainers. * - * Next, create the different data types (e.g. \ref AQNWB::NWB::ElectricalSeries "ElectricalSeries" or other AQNWB::NWB::TimeSeries "TimeSeries") - * that you would like to write data into. After creation, these objects are added - * to the \ref AQNWB::NWB::RecordingContainers "RecordingContainers" object so that it can manage access and data writing - * during the recording process. + * Next, create the different data types (e.g. \ref AQNWB::NWB::ElectricalSeries "ElectricalSeries" + * or other AQNWB::NWB::TimeSeries "TimeSeries") that you would like to write data into. After + * creation, these objects are added to the \ref AQNWB::NWB::RecordingContainers "RecordingContainers" + * object so that it can manage access and data writing during the recording process. * * \snippet tests/examples/testWorkflowExamples.cpp example_workflow_datasets_snippet * @@ -72,10 +78,10 @@ * * \subsection write_data 6. Write data. * - * During the recording process, use the \ref AQNWB::NWB::RecordingContainers "RecordingContainers" as an interface - * to access the various \ref AQNWB::NWB::Container "Container" object and corresponding - * datasets and write blocks of data to the file. Calling `flush()` on the I/O object at anytime will ensure the - * data is moved to disk. + * During the recording process, use the \ref AQNWB::NWB::RecordingContainers "RecordingContainers" + * as an interface to access the various \ref AQNWB::NWB::Container "Container" object and corresponding + * datasets and write blocks of data to the file. Calling `flush()` on the I/O object at any time will + * ensure the data is moved to disk. * * \snippet tests/examples/testWorkflowExamples.cpp example_workflow_write_snippet * From 78ffe9296aaf4cc9bbf0e717f6599bf01aefc280 Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Fri, 30 Aug 2024 17:14:35 -0700 Subject: [PATCH 09/13] Update workflow.dox --- docs/pages/userdocs/workflow.dox | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/pages/userdocs/workflow.dox b/docs/pages/userdocs/workflow.dox index ad02aafa..470442aa 100644 --- a/docs/pages/userdocs/workflow.dox +++ b/docs/pages/userdocs/workflow.dox @@ -13,7 +13,7 @@ * 2. Create the \ref AQNWB::NWB::RecordingContainers "RecordingContainers" object * used for managing \ref AQNWB::NWB::Container "Container" objects for storing recordings. * 3. Create the \ref AQNWB::NWB::NWBFile "NWBFile" object used for managing and creating NWB - * file contents. + * file contents. * 4. Create the \ref AQNWB::NWB::Container "Container" objects (e.g., * \ref AQNWB::NWB::ElectricalSeries "ElectricalSeries") used for recording and add them * to the \ref AQNWB::NWB::RecordingContainers "RecordingContainers". @@ -26,9 +26,9 @@ * * \subsection create_io 1. Create the I/O object. * - * First, create an I/O object (e.g., \ref AQNWB::HDF5::HDF5IO "HDF5IO") used for writing data to the file. - * We provide a convenience method, \ref AQNWB::createIO "createIO" - * to create this object using one of our supported backends. For more fine-grained + * First, create an I/O object (e.g., \ref AQNWB::HDF5::HDF5IO "HDF5IO") used for writing + * data to the file. AqNWB provides the convenience method, \ref AQNWB::createIO "createIO" + * to create this object using one of the supported backends. For more fine-grained * control of different backend parameters, you can create your own `std::shared_ptr` * using any of the derived \ref AQNWB::BaseIO "BaseIO" classes. * From 21fc094ed5b2d5df34e14f457688d55359a9e5e0 Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Fri, 30 Aug 2024 21:21:40 -0700 Subject: [PATCH 10/13] Fix documentation error --- src/nwb/RecordingContainers.hpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/nwb/RecordingContainers.hpp b/src/nwb/RecordingContainers.hpp index c2fe0b40..d0308140 100644 --- a/src/nwb/RecordingContainers.hpp +++ b/src/nwb/RecordingContainers.hpp @@ -72,7 +72,8 @@ class RecordingContainers * @param containerInd The index of the electrical series dataset within the * electrical series group. * @param channel The channel index to use for writing timestamps. - * @param dataShape The size of the data block. + * @param numSamples Number of samples in the time, i.e., the size of the + * first dimension of the data parameter * @param data A pointer to the data block. * @param timestamps A pointer to the timestamps block. May be null if * multidimensional TimeSeries and only need to write the timestamps once but From d1bcc725e7240b235ec80bbd9636bda2d2c40a1f Mon Sep 17 00:00:00 2001 From: Steph Prince <40640337+stephprince@users.noreply.github.com> Date: Tue, 3 Sep 2024 10:42:54 -0700 Subject: [PATCH 11/13] rename addData to addContainer --- src/nwb/RecordingContainers.cpp | 4 ++-- src/nwb/RecordingContainers.hpp | 9 ++++++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/nwb/RecordingContainers.cpp b/src/nwb/RecordingContainers.cpp index a0a5bc5a..d7464bca 100644 --- a/src/nwb/RecordingContainers.cpp +++ b/src/nwb/RecordingContainers.cpp @@ -11,9 +11,9 @@ RecordingContainers::RecordingContainers() {} RecordingContainers::~RecordingContainers() {} -void RecordingContainers::addData(std::unique_ptr data) +void RecordingContainers::addContainer(std::unique_ptr container) { - this->containers.push_back(std::move(data)); + this->containers.push_back(std::move(container)); } Container* RecordingContainers::getContainer(const SizeType& containerInd) diff --git a/src/nwb/RecordingContainers.hpp b/src/nwb/RecordingContainers.hpp index d0308140..aa003086 100644 --- a/src/nwb/RecordingContainers.hpp +++ b/src/nwb/RecordingContainers.hpp @@ -36,10 +36,13 @@ class RecordingContainers ~RecordingContainers(); /** - * @brief Adds a Container object to the container. - * @param data The Container object to add. + * @brief Adds a Container object to the container. Note that this function + * transfers ownership of the Container object to the RecordingContainers + * object, and should be called with the pattern + * recordingContainers.addContainer(std::move(container)). + * @param container The Container object to add. */ - void addData(std::unique_ptr data); + void addContainer(std::unique_ptr container); /** * @brief Gets the Container object from the recordingContainers From 1a67791cd943e1976254d9c640484147374bdbde Mon Sep 17 00:00:00 2001 From: Steph Prince <40640337+stephprince@users.noreply.github.com> Date: Tue, 3 Sep 2024 10:54:00 -0700 Subject: [PATCH 12/13] add example to track container index --- src/nwb/NWBFile.cpp | 8 ++++++-- src/nwb/NWBFile.hpp | 6 +++++- tests/examples/testWorkflowExamples.cpp | 11 +++++++---- 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/src/nwb/NWBFile.cpp b/src/nwb/NWBFile.cpp index 5d519944..007bee3c 100644 --- a/src/nwb/NWBFile.cpp +++ b/src/nwb/NWBFile.cpp @@ -23,6 +23,8 @@ using namespace AQNWB::NWB; constexpr SizeType CHUNK_XSIZE = 2048; +std::vector NWBFile::emptyContainerIndexes = {}; + // NWBFile NWBFile::NWBFile(const std::string& idText, std::shared_ptr io) @@ -93,7 +95,8 @@ Status NWBFile::createFileStructure() Status NWBFile::createElectricalSeries( std::vector recordingArrays, const BaseDataType& dataType, - RecordingContainers* recordingContainers) + RecordingContainers* recordingContainers, + std::vector& containerIndexes) { if (!io->canModifyObjects()) { return Status::Failure; @@ -132,7 +135,8 @@ Status NWBFile::createElectricalSeries( SizeArray {0, channelVector.size()}, SizeArray {CHUNK_XSIZE, 0}); electricalSeries->initialize(); - recordingContainers->addData(std::move(electricalSeries)); + recordingContainers->addContainer(std::move(electricalSeries)); + containerIndexes.push_back(recordingContainers->containers.size() - 1); // Add electrode information to electrode table (does not write to datasets // yet) diff --git a/src/nwb/NWBFile.hpp b/src/nwb/NWBFile.hpp index 61ecca04..277f45ab 100644 --- a/src/nwb/NWBFile.hpp +++ b/src/nwb/NWBFile.hpp @@ -69,13 +69,16 @@ class NWBFile * record from. A separate ElectricalSeries will be * created for each ChannelVector. * @param recordingContainers The container to store the created TimeSeries. + * @param containerIndexes The indexes of the containers added to + * recordingContainers * @param dataType The data type of the elements in the data block. * @return Status The status of the object creation operation. */ Status createElectricalSeries( std::vector recordingArrays, const BaseDataType& dataType = BaseDataType::I16, - RecordingContainers* recordingContainers = nullptr); + RecordingContainers* recordingContainers = nullptr, + std::vector& containerIndexes = emptyContainerIndexes); protected: /** @@ -120,6 +123,7 @@ class NWBFile const std::string identifierText; std::shared_ptr io; + static std::vector emptyContainerIndexes; }; } // namespace AQNWB::NWB \ No newline at end of file diff --git a/tests/examples/testWorkflowExamples.cpp b/tests/examples/testWorkflowExamples.cpp index 759ec4da..b04fd73f 100644 --- a/tests/examples/testWorkflowExamples.cpp +++ b/tests/examples/testWorkflowExamples.cpp @@ -48,8 +48,11 @@ TEST_CASE("workflowExamples") // [example_workflow_nwbfile_snippet] // [example_workflow_datasets_snippet] - nwbfile->createElectricalSeries( - mockRecordingArrays, BaseDataType::I16, recordingContainers.get()); + std::vector containerIndexes; + nwbfile->createElectricalSeries(mockRecordingArrays, + BaseDataType::I16, + recordingContainers.get(), + containerIndexes); // [example_workflow_datasets_snippet] // [example_workflow_start_snippet] @@ -60,7 +63,7 @@ TEST_CASE("workflowExamples") bool isRecording = true; while (isRecording) { // write data to the file for each channel - for (SizeType i = 0; i < mockRecordingArrays.size(); ++i) { + for (SizeType i = 0; i < containerIndexes.size(); ++i) { const auto& channelVector = mockRecordingArrays[i]; for (const auto& channel : channelVector) { // copy data into buffer @@ -80,7 +83,7 @@ TEST_CASE("workflowExamples") dataBuffer.size(), channel.getBitVolts(), dataBuffer.data()); // [example_workflow_write_snippet] - recordingContainers->writeTimeseriesData(i, + recordingContainers->writeTimeseriesData(containerIndexes[i], channel, dataShape, positionOffset, From 46059f78cbe572f505e199cbbb8190b7ef1ad6c7 Mon Sep 17 00:00:00 2001 From: Steph Prince <40640337+stephprince@users.noreply.github.com> Date: Tue, 3 Sep 2024 10:55:22 -0700 Subject: [PATCH 13/13] add docs page for container index --- docs/pages/userdocs/workflow.dox | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/pages/userdocs/workflow.dox b/docs/pages/userdocs/workflow.dox index 470442aa..106f46ff 100644 --- a/docs/pages/userdocs/workflow.dox +++ b/docs/pages/userdocs/workflow.dox @@ -57,7 +57,12 @@ * Next, create the different data types (e.g. \ref AQNWB::NWB::ElectricalSeries "ElectricalSeries" * or other AQNWB::NWB::TimeSeries "TimeSeries") that you would like to write data into. After * creation, these objects are added to the \ref AQNWB::NWB::RecordingContainers "RecordingContainers" - * object so that it can manage access and data writing during the recording process. + * object so that it can mana ge access and data writing during the recording process. + * When adding containers, ownership of the \ref AQNWB::NWB::Container "Container" is transferred to the + * \ref AQNWB::NWB::RecordingContainers "RecordingContainers" object, so that we can access it again via + * its index. New containers will always be appended to the end of the + * \ref AQNWB::NWB::RecordingContainers::containers "containers" object and their index can be tracked + * using the size of the input `recordingArrays`. * * \snippet tests/examples/testWorkflowExamples.cpp example_workflow_datasets_snippet *