From e21c02aac6d1c0c408f53169c3d23600ea5112c8 Mon Sep 17 00:00:00 2001 From: Steph Prince <40640337+stephprince@users.noreply.github.com> Date: Fri, 6 Sep 2024 12:11:27 -0700 Subject: [PATCH 1/6] update float to int transform --- src/Channel.hpp | 6 +++--- src/Utils.hpp | 28 ++++++++++++++++++++++------ tests/testRecordingWorkflow.cpp | 11 ++++------- tests/testUtils.hpp | 2 +- 4 files changed, 30 insertions(+), 17 deletions(-) diff --git a/src/Channel.hpp b/src/Channel.hpp index fccdf227..4166aa8b 100644 --- a/src/Channel.hpp +++ b/src/Channel.hpp @@ -25,9 +25,9 @@ class Channel const SizeType globalIndex, const float conversion = 1e6f, // uV to V const float samplingRate = 30000.f, // placeholder - const float bitVolts = 0.000002f, // least significant bit needed to - // convert 16-bit int to volts - // currently a placeholder + const float bitVolts = 0.05f, // least significant bit needed to + // convert 16-bit int to volts + // currently a placeholder const std::array position = {0.f, 0.f, 0.f}, const std::string comments = "no comments"); diff --git a/src/Utils.hpp b/src/Utils.hpp index f98d5b30..43a00412 100644 --- a/src/Utils.hpp +++ b/src/Utils.hpp @@ -1,9 +1,13 @@ +#include #include +#include +#include #include #include #include #include +#include #include #include #include @@ -68,6 +72,23 @@ inline std::shared_ptr createIO(const std::string& type, } } +inline void convertFloatToInt16LE(const float* source, + void* dest, + int numSamples) +{ + auto maxVal = static_cast(0x7fff); + auto intData = static_cast(dest); + + for (int i = 0; i < numSamples; ++i) { + auto clampedValue = std::clamp(maxVal * source[i], -maxVal, maxVal); + auto intValue = + static_cast(static_cast(std::round(clampedValue))); + intValue = boost::endian::native_to_little(intValue); + *reinterpret_cast(intData) = intValue; + intData += 2; // destBytesPerSample is always 2 + } +} + inline std::unique_ptr transformToInt16(SizeType numSamples, float conversion_factor, const float* data) @@ -83,12 +104,7 @@ inline std::unique_ptr transformToInt16(SizeType numSamples, [multFactor](float value) { return value * multFactor; }); // convert float to int16 - std::transform( - scaledData.get(), - scaledData.get() + numSamples, - intData.get(), - [](float value) - { return static_cast(std::clamp(value, -32768.0f, 32767.0f)); }); + convertFloatToInt16LE(scaledData.get(), intData.get(), numSamples); return intData; } diff --git a/tests/testRecordingWorkflow.cpp b/tests/testRecordingWorkflow.cpp index cdcd9a9b..3a90c5fa 100644 --- a/tests/testRecordingWorkflow.cpp +++ b/tests/testRecordingWorkflow.cpp @@ -21,7 +21,7 @@ TEST_CASE("writeContinuousData", "[recording]") { // 0. setup mock data SizeType numChannels = 4; - SizeType numSamples = 300; + SizeType numSamples = 100; SizeType samplesRecorded = 0; SizeType bufferSize = numSamples / 10; std::vector dataBuffer(bufferSize); @@ -48,7 +48,7 @@ TEST_CASE("writeContinuousData", "[recording]") // 4. create datasets and add to recording containers nwbfile->createElectricalSeries( - mockRecordingArrays, BaseDataType::I16, recordingContainers.get()); + mockRecordingArrays, BaseDataType::F32, recordingContainers.get()); // 5. start the recording io->startRecording(); @@ -73,14 +73,12 @@ TEST_CASE("writeContinuousData", "[recording]") std::vector positionOffset = {samplesRecorded, channel.localIndex}; std::vector dataShape = {dataBuffer.size(), 1}; - std::unique_ptr intBuffer = transformToInt16( - dataBuffer.size(), channel.getBitVolts(), dataBuffer.data()); recordingContainers->writeTimeseriesData(i, channel, dataShape, positionOffset, - intBuffer.get(), + dataBuffer.data(), timestampsBuffer.data()); } } @@ -114,8 +112,7 @@ TEST_CASE("writeContinuousData", "[recording]") std::vector(numSamples)); for (SizeType i = 0; i < numChannelsToRead; ++i) { for (SizeType j = 0; j < numSamples; ++j) { - dataOut[i][j] = - buffer[j * numChannelsToRead + i] * (32767.0f * 0.000002f); + dataOut[i][j] = buffer[j * numChannelsToRead + i]; } } delete[] buffer; diff --git a/tests/testUtils.hpp b/tests/testUtils.hpp index 914d8459..e8acc658 100644 --- a/tests/testUtils.hpp +++ b/tests/testUtils.hpp @@ -80,7 +80,7 @@ inline std::vector> getMockData2D(SizeType numSamples = 1000, for (auto& channelData : mockData) { for (auto& data : channelData) { data = static_cast(dis(rng)) - * 1000.f; // approximate microvolt unit range + * 100.f; // approximate microvolt unit range } } From 08c44c12ba74e6b4e3f22b2aff78e47c90fcabd2 Mon Sep 17 00:00:00 2001 From: Steph Prince <40640337+stephprince@users.noreply.github.com> Date: Fri, 6 Sep 2024 12:11:52 -0700 Subject: [PATCH 2/6] update electrical series dataset write --- src/nwb/ecephys/ElectricalSeries.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/nwb/ecephys/ElectricalSeries.cpp b/src/nwb/ecephys/ElectricalSeries.cpp index ec3a3bb2..bf186d27 100644 --- a/src/nwb/ecephys/ElectricalSeries.cpp +++ b/src/nwb/ecephys/ElectricalSeries.cpp @@ -42,9 +42,11 @@ void ElectricalSeries::initialize() TimeSeries::initialize(); // setup variables based on number of channels - std::vector electrodeInds(channelVector.size()); + std::vector electrodeInds(channelVector.size()); + std::vector channelConversions(channelVector.size()); for (size_t i = 0; i < channelVector.size(); ++i) { electrodeInds[i] = channelVector[i].globalIndex; + channelConversions[i] = channelVector[i].getConversion(); } samplesRecorded = SizeArray(channelVector.size(), 0); @@ -54,6 +56,10 @@ void ElectricalSeries::initialize() SizeArray {1}, chunkSize, getPath() + "/channel_conversion")); + channelConversion->writeDataBlock( + std::vector(1, channelVector.size()), + BaseDataType::F32, + &channelConversions[0]); io->createCommonNWBAttributes(getPath() + "/channel_conversion", "hdmf-common", "", From cb7552d633dee3f6c8ad9ce0b61fa42ae75d4f7d Mon Sep 17 00:00:00 2001 From: Steph Prince <40640337+stephprince@users.noreply.github.com> Date: Fri, 6 Sep 2024 12:12:02 -0700 Subject: [PATCH 3/6] update tests --- tests/reader.cpp | 62 ++++++++++++++++++++++++++---------------------- 1 file changed, 33 insertions(+), 29 deletions(-) diff --git a/tests/reader.cpp b/tests/reader.cpp index 1809c21a..75b65c02 100644 --- a/tests/reader.cpp +++ b/tests/reader.cpp @@ -9,35 +9,39 @@ using namespace H5; int readerFunction(const std::string& path, const std::string& dataPath) { - std::unique_ptr file = - std::make_unique(path, H5F_ACC_RDONLY | H5F_ACC_SWMR_READ); - std::unique_ptr dSet = - std::make_unique(file->openDataSet(dataPath)); - - std::vector dsetSizes; - for (int i = 0; i < 3; ++i) { - H5Drefresh(dSet->getId()); - - // Get the current size of the dataset - DataSpace fSpace = dSet->getSpace(); - hsize_t currentSize; - fSpace.getSimpleExtentDims(¤tSize, nullptr); - - // Update the size - dsetSizes.push_back(currentSize); - sleep(1); // Simulate real-time data streaming - } - - // print out dataset sizes - std::cout << "Dataset sizes: "; - for (int val : dsetSizes) { - std::cout << val << " "; - } - std::cout << std::endl; - - // check that data is being appended (last value should be greater than the - // first) - if (dsetSizes[0] >= dsetSizes[2]) { + try { + std::unique_ptr file = + std::make_unique(path, H5F_ACC_RDONLY | H5F_ACC_SWMR_READ); + std::unique_ptr dSet = + std::make_unique(file->openDataSet(dataPath)); + + std::vector dsetSizes; + for (int i = 0; i < 3; ++i) { + H5Drefresh(dSet->getId()); + + // Get the current size of the dataset + DataSpace fSpace = dSet->getSpace(); + hsize_t currentSize; + fSpace.getSimpleExtentDims(¤tSize, nullptr); + + // Update the size + dsetSizes.push_back(currentSize); + sleep(1); // Simulate real-time data streaming + } + + // print out dataset sizes + std::cout << "Dataset sizes: "; + for (int val : dsetSizes) { + std::cout << val << " "; + } + std::cout << std::endl; + + // check that data is being appended (last value should be greater than the + // first) + if (dsetSizes[0] >= dsetSizes[2]) { + return -1; + } + } catch (const FileIException& error) { return -1; } From 6a913bbcf7d10055fd486b1979c1f7b69b9e4eca Mon Sep 17 00:00:00 2001 From: Steph Prince <40640337+stephprince@users.noreply.github.com> Date: Fri, 6 Sep 2024 12:12:14 -0700 Subject: [PATCH 4/6] update description and data collection inputs --- src/nwb/NWBFile.cpp | 11 +++++++++-- src/nwb/NWBFile.hpp | 8 +++++++- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/src/nwb/NWBFile.cpp b/src/nwb/NWBFile.cpp index 007bee3c..5ce0ee14 100644 --- a/src/nwb/NWBFile.cpp +++ b/src/nwb/NWBFile.cpp @@ -35,8 +35,12 @@ NWBFile::NWBFile(const std::string& idText, std::shared_ptr io) NWBFile::~NWBFile() {} -Status NWBFile::initialize() +Status NWBFile::initialize(const std::string description, + const std::string dataCollection) { + this->description = description; + this->dataCollection = dataCollection; + if (std::filesystem::exists(io->getFileName())) { return io->open(false); } else { @@ -68,6 +72,9 @@ Status NWBFile::createFileStructure() io->createGroup("/general"); io->createGroup("/general/devices"); io->createGroup("/general/extracellular_ephys"); + if (dataCollection != "") { + io->createStringDataSet("/general/data_collection", dataCollection); + } io->createGroup("/specifications"); io->createReferenceAttribute("/specifications", "/", ".specloc"); @@ -84,7 +91,7 @@ Status NWBFile::createFileStructure() std::string time = getCurrentTime(); std::vector timeVec = {time}; io->createStringDataSet("/file_create_date", timeVec); - io->createStringDataSet("/session_description", "a recording session"); + io->createStringDataSet("/session_description", description); io->createStringDataSet("/session_start_time", time); io->createStringDataSet("/timestamps_reference_time", time); io->createStringDataSet("/identifier", identifierText); diff --git a/src/nwb/NWBFile.hpp b/src/nwb/NWBFile.hpp index 277f45ab..2c13cdf8 100644 --- a/src/nwb/NWBFile.hpp +++ b/src/nwb/NWBFile.hpp @@ -51,8 +51,11 @@ class NWBFile /** * @brief Initializes the NWB file by opening and setting up the file * structure. + * @param description A description of the NWBFile session. + * @param dataCollection Information about the data collection methods. */ - Status initialize(); + Status initialize(const std::string description = "a recording session", + const std::string dataCollection = ""); /** * @brief Finalizes the NWB file by closing it. @@ -124,6 +127,9 @@ class NWBFile const std::string identifierText; std::shared_ptr io; static std::vector emptyContainerIndexes; + + std::string description; + std::string dataCollection; }; } // namespace AQNWB::NWB \ No newline at end of file From cb6257e3ca9a08ac790786c08f610e02b5743811 Mon Sep 17 00:00:00 2001 From: Steph Prince <40640337+stephprince@users.noreply.github.com> Date: Fri, 6 Sep 2024 16:54:41 -0700 Subject: [PATCH 5/6] do not save description and dataCollection on nwbfile --- src/nwb/NWBFile.cpp | 8 +++----- src/nwb/NWBFile.hpp | 8 ++++---- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/src/nwb/NWBFile.cpp b/src/nwb/NWBFile.cpp index 5ce0ee14..361f298d 100644 --- a/src/nwb/NWBFile.cpp +++ b/src/nwb/NWBFile.cpp @@ -38,14 +38,11 @@ NWBFile::~NWBFile() {} Status NWBFile::initialize(const std::string description, const std::string dataCollection) { - this->description = description; - this->dataCollection = dataCollection; - if (std::filesystem::exists(io->getFileName())) { return io->open(false); } else { io->open(true); - return createFileStructure(); + return createFileStructure(description, dataCollection); } } @@ -54,7 +51,8 @@ Status NWBFile::finalize() return io->close(); } -Status NWBFile::createFileStructure() +Status NWBFile::createFileStructure(std::string description, + std::string dataCollection) { if (!io->canModifyObjects()) { return Status::Failure; diff --git a/src/nwb/NWBFile.hpp b/src/nwb/NWBFile.hpp index 2c13cdf8..313c54e2 100644 --- a/src/nwb/NWBFile.hpp +++ b/src/nwb/NWBFile.hpp @@ -89,9 +89,12 @@ class NWBFile * Note, this function will fail if the file is in a mode where * new objects cannot be added, which can be checked via * nwbfile.io->canModifyObjects() + * @param description A description of the NWBFile session. + * @param dataCollection Information about the data collection methods. * @return Status The status of the file structure creation. */ - Status createFileStructure(); + Status createFileStructure(std::string description, + std::string dataCollection); private: /** @@ -127,9 +130,6 @@ class NWBFile const std::string identifierText; std::shared_ptr io; static std::vector emptyContainerIndexes; - - std::string description; - std::string dataCollection; }; } // namespace AQNWB::NWB \ No newline at end of file From 8a6b77690006e95ff044c679486b2d0d57d2ac97 Mon Sep 17 00:00:00 2001 From: Steph Prince <40640337+stephprince@users.noreply.github.com> Date: Fri, 6 Sep 2024 16:54:56 -0700 Subject: [PATCH 6/6] add notes to data transform utils --- src/Utils.hpp | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/Utils.hpp b/src/Utils.hpp index 43a00412..5c75f98c 100644 --- a/src/Utils.hpp +++ b/src/Utils.hpp @@ -72,10 +72,21 @@ inline std::shared_ptr createIO(const std::string& type, } } +/** + * @brief Method to convert float values to uint16 values. This method + * was adapted from JUCE AudioDataConverters using a default value of + * destBytesPerSample = 2. + * @param source The source float data to convert + * @param dest The destination for the converted uint16 data + * @param numSamples The number of samples to convert + */ inline void convertFloatToInt16LE(const float* source, void* dest, int numSamples) { + // TODO - several steps in this function may be unnecessary for our use + // case. Consider simplifying the intermediate cast to char and the + // final cast to uint16_t. auto maxVal = static_cast(0x7fff); auto intData = static_cast(dest); @@ -89,6 +100,12 @@ inline void convertFloatToInt16LE(const float* source, } } +/** + * @brief Method to scale float values and convert to int16 values + * @param numSamples The number of samples to convert + * @param conversion_factor The conversion factor to scale the data + * @param data The data to convert + */ inline std::unique_ptr transformToInt16(SizeType numSamples, float conversion_factor, const float* data)