diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..d507b6f --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "Source/aqnwb"] + path = Source/aqnwb + url = https://github.com/NeurodataWithoutBorders/aqnwb diff --git a/CMakeLists.txt b/CMakeLists.txt index 9b947a1..f5e81db 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -33,9 +33,11 @@ set_property(DIRECTORY APPEND PROPERTY COMPILE_DEFINITIONS set(SOURCE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/Source) -file(GLOB_RECURSE SRC_FILES LIST_DIRECTORIES false "${SOURCE_PATH}/*.cpp" "${SOURCE_PATH}/*.h") +file(GLOB_RECURSE SRC_FILES LIST_DIRECTORIES false "${SOURCE_PATH}/*.cpp" "${SOURCE_PATH}/*.h" "${SOURCE_PATH}/*.hpp") set(GUI_COMMONLIB_DIR ${GUI_BASE_DIR}/installed_libs) +include_directories(${SOURCE_PATH}/aqnwb/src) + set(CONFIGURATION_FOLDER $<$:Debug>$<$>:Release>) list(APPEND CMAKE_PREFIX_PATH ${GUI_COMMONLIB_DIR} ${GUI_COMMONLIB_DIR}/${CONFIGURATION_FOLDER}) @@ -48,6 +50,7 @@ endif() target_compile_features(${PLUGIN_NAME} PUBLIC cxx_auto_type cxx_generalized_initializers cxx_std_17) target_include_directories(${PLUGIN_NAME} PUBLIC ${GUI_BASE_DIR}/JuceLibraryCode ${GUI_BASE_DIR}/JuceLibraryCode/modules ${GUI_BASE_DIR}/Plugins/Headers ${GUI_COMMONLIB_DIR}/include) +target_compile_definitions(${PLUGIN_NAME} PUBLIC BOOST_NO_CXX98_FUNCTION_BASE) set(GUI_BIN_DIR ${GUI_BASE_DIR}/Build/${CONFIGURATION_FOLDER}) @@ -110,8 +113,10 @@ endif() include_directories(${HDF5_INCLUDE_DIRS}) -target_link_libraries(${PLUGIN_NAME} ${HDF5_LIBRARIES} ${HDF5_CXX_LIBRARIES}) -target_include_directories(${PLUGIN_NAME} PRIVATE ${HDF5_INCLUDE_DIRS}) +find_package(Boost REQUIRED) + +target_link_libraries(${PLUGIN_NAME} ${HDF5_LIBRARIES} ${HDF5_CXX_LIBRARIES} ${Boost_LIBRARIES}) +target_include_directories(${PLUGIN_NAME} PRIVATE ${HDF5_INCLUDE_DIRS} ${Boost_INCLUDE_DIRS}) #target_include_directories(${PLUGIN_NAME} PUBLIC ../OpenEphysHDF5Lib/Source) diff --git a/Source/RecordEngine/NWBFormat.cpp b/Source/RecordEngine/NWBFormat.cpp deleted file mode 100644 index 32bfcc8..0000000 --- a/Source/RecordEngine/NWBFormat.cpp +++ /dev/null @@ -1,946 +0,0 @@ -/* - ------------------------------------------------------------------ - - This file is part of the Open Ephys GUI - Copyright (C) 2014 Open Ephys - - ------------------------------------------------------------------ - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . - - */ - - #include "NWBFormat.h" - - using namespace NWBRecording; - -#ifndef EVENT_CHUNK_SIZE -#define EVENT_CHUNK_SIZE 8 -#endif - -#ifndef SPIKE_CHUNK_XSIZE -#define SPIKE_CHUNK_XSIZE 8 -#endif - -#ifndef SPIKE_CHUNK_YSIZE -#define SPIKE_CHUNK_YSIZE 40 -#endif - - #define MAX_BUFFER_SIZE 40960 - -NWBFile::NWBFile(String fName, String ver, String idText) : - HDF5FileBase(), - filename(fName), - identifierText(idText), - GUIVersion(ver) -{ - readyToOpen = true; //In KWIK this is in initFile, but the new recordEngine methods make it safe for it to be here - - scaledBuffer.malloc(MAX_BUFFER_SIZE); - intBuffer.malloc(MAX_BUFFER_SIZE); - bufferSize = MAX_BUFFER_SIZE; -} - -NWBFile::~NWBFile() -{ - continuousDataSets.clear(); - spikeDataSets.clear(); - eventDataSets.clear(); - syncMsgDataSet.reset(); -} - -int NWBFile::createFileStructure() -{ - - setAttributeStr("core", "/", "namespace"); - setAttributeStr("NWBFile", "/", "neurodata_type"); - setAttributeStr("2.5.0", "/", "nwb_version"); - setAttributeStr(identifierText, "/", "object_id"); - - if (createGroup("/acquisition")) return -1; - - if (createGroup("/analysis")) return -1; - - String time = Time::getCurrentTime().formatted("%Y-%m-%dT%H:%M:%S") + Time::getCurrentTime().getUTCOffsetString(true); - - createTextDataSet("", "file_create_date", time); - - if (createGroup("/general")) return -1; - if (createGroup("general/devices")) return -1; - if (createGroup("general/extracellular_ephys")) return -1; - if (createGroup("general/extracellular_ephys/electrodes")) return -1; - - StringArray colnames; - colnames.add("group"); - colnames.add("group_name"); - colnames.add("location"); - setAttributeStrArray(colnames, "general/extracellular_ephys/electrodes", "colnames"); - setAttributeStr("metadata about extracellular electrodes", "general/extracellular_ephys/electrodes", "description"); - setAttributeStr("hdmf-common", "general/extracellular_ephys/electrodes", "namespace"); - setAttributeStr("DynamicTable", "general/extracellular_ephys/electrodes", "neurodata_type"); - setAttributeStr(generateUuid(), "general/extracellular_ephys/electrodes", "object_id"); - - if (createGroup("/processing")) return -1; - - if (createGroup("/stimulus")) return -1; - if (createGroup("/stimulus/presentation")) return -1; - if (createGroup("/stimulus/templates")) return -1; - - createStringDataSet("/session_description", "Recording with the Open Ephys GUI"); - createStringDataSet("/session_start_time", time); - createStringDataSet("/timestamps_reference_time", time); - createStringDataSet("/identifier", "test-identifier"); - - return 0; - -} - -TimeSeries::TimeSeries(String rootPath, String name, String description_) - : basePath(rootPath + name), description(description_) -{ - -} - -ecephys::ElectricalSeries::ElectricalSeries(String rootPath, String name, String description_, - int channel_count_, Array channel_conversion_) - : TimeSeries(rootPath, name, description_), - channel_conversion(channel_conversion_), - channel_count(channel_count_) -{ - -} - -ecephys::SpikeEventSeries::SpikeEventSeries(String rootPath, String name, String description_, - int channel_count, Array channel_conversion_) - : ecephys::ElectricalSeries(rootPath, name, description_, channel_count, channel_conversion_) -{ - -} - -TTLEventSeries::TTLEventSeries(String rootPath, String name, String description_) - : TimeSeries(rootPath, name, description_) -{ - -} - -AnnotationSeries::AnnotationSeries(String rootPath, String name, String description_) - : TimeSeries(rootPath, name, description_) -{ - -} - - -bool NWBFile::startNewRecording( - int recordingNumber, - const Array& continuousArray, - const Array& continuousChannels, - const Array& eventArray, - const Array& electrodeArray) -{ - - // all recorded data is stored in the "acquisition" group - String rootPath = "/acquisition/"; - - continuousDataSets.clearQuick(true); - spikeDataSets.clearQuick(true); - eventDataSets.clearQuick(true); - - Array all_electrode_inds; - StringArray groupNames; - StringArray groupReferences; - - // 0. put global inds into electrode table - for (auto ch : continuousChannels) - { - all_electrode_inds.add(ch->getGlobalIndex()); - - String groupName = ch->getSourceNodeName() + "-" - + String(ch->getSourceNodeId()) - + "." + ch->getStreamName(); - - groupNames.add(groupName); - groupReferences.add("/general/extracellular_ephys/" + groupName); - } - - // 1. Create continuous datasets - for (int i = 0; i < continuousArray.size(); i++) - { - - // Get the scaling info for each channel - ContinuousGroup group = continuousArray.getReference(i); - - Array channel_conversion; - for (int ch = 0; ch < group.size(); ch++) - { - channel_conversion.add(group[ch]->getBitVolts() / 1e6); - } - - String groupName = group[0]->getSourceNodeName() + "-" - + String(group[0]->getSourceNodeId()) - + "." + group[0]->getStreamName(); - - String fullPath = "general/extracellular_ephys/" + groupName; - createGroup(fullPath); - setAttributeStr("description", fullPath, "description"); - setAttributeStr("unknown", fullPath, "location"); - setAttributeStr("core", fullPath, "namespace"); - setAttributeStr("ElectrodeGroup", fullPath, "neurodata_type"); - setAttributeStr(generateUuid(), fullPath, "object_id"); - - createGroup("general/devices/" + groupName); - - setAttributeStr("description", "general/devices/" + groupName, "description"); - setAttributeStr("unknown", "general/devices/" + groupName, "manufacturer"); - setAttributeStr("core", "general/devices/" + groupName, "namespace"); - setAttributeStr("Device", "general/devices/" + groupName, "neurodata_type"); - setAttributeStr(generateUuid(), "general/devices/" + groupName, "object_id"); - - createReference("/" + fullPath + "/device", "/general/devices/" + groupName); - - Array electrode_inds; - for (int ch = 0; ch < group.size(); ch++) - { - int index = group[ch]->getGlobalIndex(); - electrode_inds.add(index); - } - - ecephys::ElectricalSeries* electricalSeries = - new ecephys::ElectricalSeries(rootPath, - groupName, - "Stores continuously sampled voltage data from an extracellular ephys recording", - group.size(), - channel_conversion - ); - - if (recordingNumber == 0) - if (!createTimeSeriesBase(electricalSeries)) - return false; - - electricalSeries->baseDataSet = createDataSet(BaseDataType::I16, - 0, - electricalSeries->channel_count, - CHUNK_XSIZE, - electricalSeries->basePath + "/data"); - - if (electricalSeries->baseDataSet == nullptr) - { - std::cerr << "Error creating dataset for " << groupName << std::endl; - return false; - } else { - createDataAttributes(electricalSeries->basePath, channel_conversion[0], -1.0f, "volts"); - } - - electricalSeries->timestampDataSet = - createTimestampDataSet(electricalSeries->basePath + "/timestamps", CHUNK_XSIZE, 1/group[0]->getSampleRate()); - if (electricalSeries->timestampDataSet == nullptr) return false; - - electricalSeries->sampleNumberDataSet = - createSampleNumberDataSet(electricalSeries->basePath + "/sync", CHUNK_XSIZE); - if (electricalSeries->sampleNumberDataSet == nullptr) return false; - - electricalSeries->channelConversionDataSet = createChannelConversionDataSet(electricalSeries->basePath + "/channel_conversion", "Bit volts values for all channels", CHUNK_XSIZE); - - if (electricalSeries->channelConversionDataSet == nullptr) return false; - writeChannelConversions(electricalSeries); - - electricalSeries->electrodeDataSet = createElectrodeDataSet(electricalSeries->basePath + "/electrodes", "Electrode index for each channel", CHUNK_XSIZE); - - if (electricalSeries->electrodeDataSet == nullptr) return false; - writeElectrodes(electricalSeries, electrode_inds); - - continuousDataSets.add(electricalSeries); - } - - // 2. create spike datasets - for (int i = 0; i < electrodeArray.size(); i++) - { - const SpikeChannel* sourceInfo = electrodeArray[i]; - - String sourceName = sourceInfo->getSourceNodeName() + "-" + String(sourceInfo->getSourceNodeId()); - sourceName += "." + sourceInfo->getStreamName(); - sourceName += "." + sourceInfo->getName(); - - Array channel_conversion; - - for (int ch = 0; ch < sourceInfo->getNumChannels(); ch++) - { - channel_conversion.add(sourceInfo->getSourceChannels()[0]->getBitVolts() / 1e6); - } - - Array electrode_inds; - - for (int ch = 0; ch < sourceInfo->getNumChannels(); ch++) - { - int globalIndex = sourceInfo->getSourceChannels()[ch]->getGlobalIndex(); - - electrode_inds.add(globalIndex); - } - - ecephys::SpikeEventSeries* spikeEventSeries = - new ecephys::SpikeEventSeries(rootPath, sourceName, - "Stores spike waveforms from an extracellular ephys recording", - sourceInfo->getNumChannels(), - channel_conversion); - - if (recordingNumber == 0) - if (!createTimeSeriesBase(spikeEventSeries)) - return false; - - spikeEventSeries->baseDataSet = createDataSet(BaseDataType::I16, 0, sourceInfo->getNumChannels(), sourceInfo->getTotalSamples(), SPIKE_CHUNK_XSIZE, spikeEventSeries->basePath + "/data"); - - if (spikeEventSeries->baseDataSet == nullptr) - { - std::cerr << "Error creating dataset for electrode " << i << std::endl; - return false; - } else { - createDataAttributes(spikeEventSeries->basePath, channel_conversion[0], -1.0f, "volts"); - } - - spikeEventSeries->timestampDataSet = - createTimestampDataSet(spikeEventSeries->basePath + "/timestamps", CHUNK_XSIZE, 1/sourceInfo->getSourceChannels()[0]->getSampleRate()); - if (spikeEventSeries->timestampDataSet == nullptr) return false; - - spikeEventSeries->sampleNumberDataSet = - createSampleNumberDataSet(spikeEventSeries->basePath + "/sync", CHUNK_XSIZE); - if (spikeEventSeries->sampleNumberDataSet == nullptr) return false; - - spikeEventSeries->channelConversionDataSet = createChannelConversionDataSet(spikeEventSeries->basePath + "/channel_conversion", "Bit volts values for all channels", CHUNK_XSIZE); - - if (spikeEventSeries->channelConversionDataSet == nullptr) return false; - writeChannelConversions(spikeEventSeries); - - spikeEventSeries->electrodeDataSet = createElectrodeDataSet(spikeEventSeries->basePath + "/electrodes", "Electrode index for each channel", CHUNK_XSIZE); - - if (spikeEventSeries->electrodeDataSet == nullptr) return false; - writeElectrodes(spikeEventSeries, electrode_inds); - - spikeDataSets.add(spikeEventSeries); - - } - - // 3. Create event channel datasets - for (int i = 0; i < eventArray.size(); i++) - { - - const EventChannel* info = eventArray[i]; - - String sourceName = info->getSourceNodeName() + "-" + String(info->getSourceNodeId()); - sourceName += "." + info->getStreamName(); - - String typeString, description; - - if (info->getType() == EventChannel::TTL) - { - TTLEventSeries* ttlEventSeries = - new TTLEventSeries(rootPath, sourceName + ".TTL", "Stores the times and lines of TTL events"); - - if (recordingNumber == 0) - if (!createTimeSeriesBase(ttlEventSeries)) return false; - - ttlEventSeries->baseDataSet = createDataSet(getEventH5Type(info->getType(), info->getLength()), 0, EVENT_CHUNK_SIZE, ttlEventSeries->basePath + "/data"); - - if (ttlEventSeries->baseDataSet == nullptr) - { - std::cerr << "Error creating dataset for event " << info->getName() << std::endl; - return false; - } - - ttlEventSeries->timestampDataSet = - createTimestampDataSet(ttlEventSeries->basePath + "/timestamps", EVENT_CHUNK_SIZE, 1/info->getSampleRate()); - if (ttlEventSeries->timestampDataSet == nullptr) return false; - - ttlEventSeries->sampleNumberDataSet = createSampleNumberDataSet(ttlEventSeries->basePath + "/sync", EVENT_CHUNK_SIZE); - if (ttlEventSeries->sampleNumberDataSet == nullptr) return false; - - ttlEventSeries->ttlWordDataSet = createDataSet(BaseDataType::U64, 0, info->getDataSize(), EVENT_CHUNK_SIZE, ttlEventSeries->basePath + "/full_word"); - if (ttlEventSeries->ttlWordDataSet == nullptr) return false; - - eventDataSets.add(ttlEventSeries); - - } - else if (info->getType() == EventChannel::TEXT) - { - AnnotationSeries* annotationSeries = new AnnotationSeries(rootPath, "messages", "Stores timestamped messages generated during an experiment"); - - if (recordingNumber == 0) - if (!createTimeSeriesBase(annotationSeries)) return false; - - annotationSeries->baseDataSet = createDataSet(getEventH5Type(info->getType(), info->getLength()), 0, EVENT_CHUNK_SIZE, annotationSeries->basePath + "/data"); - - if (annotationSeries->baseDataSet == nullptr) - { - std::cerr << "Error creating dataset for event " << info->getName() << std::endl; - return false; - } - - annotationSeries->timestampDataSet = createTimestampDataSet(annotationSeries->basePath + "/timestamps", EVENT_CHUNK_SIZE, 1/info->getSampleRate()); - if (annotationSeries->timestampDataSet == nullptr) return false; - - annotationSeries->sampleNumberDataSet = createSampleNumberDataSet(annotationSeries->basePath + "/sync", EVENT_CHUNK_SIZE); - if (annotationSeries->sampleNumberDataSet == nullptr) return false; - - messagesDataSet.reset(annotationSeries); - } - - } - - //4. Create sync messages dataset - String desc = "Stores recording start timestamps for each processor in text format"; - - AnnotationSeries* annotationSeries = new AnnotationSeries(rootPath, "sync_messages", desc); - - if (recordingNumber == 0) - { - if (!createTimeSeriesBase(annotationSeries)) return false; - - annotationSeries->baseDataSet = createDataSet(BaseDataType::STR(100), 0, 1, annotationSeries->basePath + "/data"); - - if (annotationSeries->baseDataSet == nullptr) - { - std::cerr << "Error creating dataset for sync messages" << std::endl; - return false; - } - } - else { - annotationSeries->baseDataSet = getDataSet(annotationSeries->basePath + "/data"); - } - - if (recordingNumber == 0) - { - annotationSeries->sampleNumberDataSet = createSampleNumberDataSet(annotationSeries->basePath + "/sync", 1); - if (annotationSeries->sampleNumberDataSet == nullptr) return false; - } - else { - annotationSeries->sampleNumberDataSet = getDataSet(annotationSeries->basePath + "/sync"); - } - - if (recordingNumber == 0) - { - annotationSeries->timestampDataSet = createTimestampDataSet(annotationSeries->basePath + "/timestamps", 1, 1); - if (annotationSeries->timestampDataSet == nullptr) return false; - } - else { - annotationSeries->timestampDataSet = getDataSet(annotationSeries->basePath + "/timestamps"); - } - - syncMsgDataSet.reset(annotationSeries); - - // 5. Create electrode table - ScopedPointer elSet = createDataSet(BaseDataType::I32, 1, 1, "general/extracellular_ephys/electrodes/id"); - - std::vector electrodeNumbers; - for (auto i : all_electrode_inds) - electrodeNumbers.push_back(i); - - CHECK_ERROR(elSet->writeDataBlock(electrodeNumbers.size(), BaseDataType::I32, &electrodeNumbers[0])); - - setAttributeStr("hdmf-common", "general/extracellular_ephys/electrodes/id", "namespace"); - setAttributeStr("ElementIdentifiers", "general/extracellular_ephys/electrodes/id", "neurodata_type"); - setAttributeStr(generateUuid(), "general/extracellular_ephys/electrodes/id", "object_id"); - - ScopedPointer groupNamesDataset = createDataSet(BaseDataType::STR(250), 0, 1, "general/extracellular_ephys/electrodes/group_name"); - - for (int i = 0; i < groupNames.size(); i++) - groupNamesDataset->writeDataBlock(1, BaseDataType::STR(groupNames[i].length()), groupNames[i].toUTF8()); - - setAttributeStr("the name of the ElectrodeGroup this electrode is a part of", "general/extracellular_ephys/electrodes/group_name", "description"); - setAttributeStr("hdmf-common", "general/extracellular_ephys/electrodes/group_name", "namespace"); - setAttributeStr("VectorData", "general/extracellular_ephys/electrodes/group_name", "neurodata_type"); - setAttributeStr(generateUuid(), "general/extracellular_ephys/electrodes/group_name", "object_id"); - - ScopedPointer locationsDataset = createDataSet(BaseDataType::STR(250), 0, 1, "general/extracellular_ephys/electrodes/location"); - - for (int i = 0; i < groupNames.size(); i++) - locationsDataset->writeDataBlock(1, BaseDataType::STR(7), String("unknown").toUTF8()); - - setAttributeStr("the location of channel within the subject e.g. brain region", "general/extracellular_ephys/electrodes/location", "description"); - setAttributeStr("hdmf-common", "general/extracellular_ephys/electrodes/location", "namespace"); - setAttributeStr("VectorData", "general/extracellular_ephys/electrodes/location", "neurodata_type"); - setAttributeStr(generateUuid(), "general/extracellular_ephys/electrodes/location", "object_id"); - - createReferenceDataSet("general/extracellular_ephys/electrodes/group", groupReferences); - - setAttributeStr("a reference to the ElectrodeGroup this electrode is a part of", "general/extracellular_ephys/electrodes/group", "description"); - setAttributeStr("hdmf-common", "general/extracellular_ephys/electrodes/group", "namespace"); - setAttributeStr("VectorData", "general/extracellular_ephys/electrodes/group", "neurodata_type"); - setAttributeStr(generateUuid(), "general/extracellular_ephys/electrodes/group", "object_id"); - - return true; - - } - - void NWBFile::stopRecording() - { - - const TimeSeries* tsStruct; - - for (int i = 0; i < continuousDataSets.size(); i++) - { - tsStruct = continuousDataSets[i]; - CHECK_ERROR(setAttribute(BaseDataType::U64, &(tsStruct->numSamples), tsStruct->basePath, "num_samples")); - } - - for (int i = 0; i < spikeDataSets.size(); i++) - { - tsStruct = spikeDataSets[i]; - CHECK_ERROR(setAttribute(BaseDataType::U64, &(tsStruct->numSamples), tsStruct->basePath, "num_samples")); - } - - for (int i = 0; i < eventDataSets.size(); i++) - { - tsStruct = eventDataSets[i]; - CHECK_ERROR(setAttribute(BaseDataType::U64, &(tsStruct->numSamples), tsStruct->basePath, "num_samples")); - } - - CHECK_ERROR(setAttribute(BaseDataType::U64, &(syncMsgDataSet->numSamples), syncMsgDataSet->basePath, "num_samples")); - } - - void NWBFile::writeData(int datasetID, int channel, int nSamples, const float* data, float bitVolts) - { - if (!continuousDataSets[datasetID]) - return; - - if (nSamples > bufferSize) //Shouldn't happen, and if it happens it'll be slow, but better this than crashing. Will be reset on file close and reset. - { - std::cerr << "Write buffer overrun, resizing to" << nSamples << std::endl; - bufferSize = nSamples; - scaledBuffer.malloc(nSamples); - intBuffer.malloc(nSamples); - } - - double multFactor = 1 / (float(0x7fff) * bitVolts); - FloatVectorOperations::copyWithMultiply(scaledBuffer.getData(), data, multFactor, nSamples); - AudioDataConverters::convertFloatToInt16LE(scaledBuffer.getData(), intBuffer.getData(), nSamples); - - continuousDataSets[datasetID]->baseDataSet->writeDataRow(channel, nSamples, BaseDataType::I16, intBuffer); - //CHECK_ERROR(); - - /* Since channels are filled asynchronouysly by the Record Thread, there is no guarantee - that at a any point in time all channels in a dataset have the same number of filled samples. - However, since each dataset is filled from a single source, all channels must have the - same number of samples at acquisition stop. To keep track of the written samples we must chose - an arbitrary channel, and at the end all channels will be the same. */ - - if (channel == 0) //there will always be a first channel or there wouldn't be dataset - continuousDataSets[datasetID]->numSamples += nSamples; - } - - void NWBFile::writeSampleNumbers(int datasetID, int nSamples, const int64* data) - { - if (!continuousDataSets[datasetID]) - return; - - CHECK_ERROR(continuousDataSets[datasetID]->sampleNumberDataSet->writeDataBlock(nSamples, BaseDataType::I64, data)); - } - - void NWBFile::writeTimestamps(int datasetID, int nSamples, const double* data) - { - if (!continuousDataSets[datasetID]) - return; - - CHECK_ERROR(continuousDataSets[datasetID]->timestampDataSet->writeDataBlock(nSamples, BaseDataType::F64, data)); - } - -void NWBFile::writeChannelConversions(ecephys::ElectricalSeries* electricalSeries) -{ - std::vector conversions; - for (auto c : electricalSeries->channel_conversion) - conversions.push_back(c); - - CHECK_ERROR(electricalSeries->channelConversionDataSet->writeDataBlock(conversions.size(), BaseDataType::F32, &conversions[0])); -} - - void NWBFile::writeElectrodes(ecephys::ElectricalSeries* electricalSeries, Array electrodeInds) - { - std::vector electrodeNumbers; - for (auto i : electrodeInds) - electrodeNumbers.push_back(i); - - CHECK_ERROR(electricalSeries->electrodeDataSet->writeDataBlock(electricalSeries->channel_count, BaseDataType::I32, &electrodeNumbers[0])); - } - - void NWBFile::writeSpike(int electrodeId, const SpikeChannel* channel, const Spike* event) - { - if (!spikeDataSets[electrodeId]) - return; - int nSamples = channel->getTotalSamples() * channel->getNumChannels(); - - if (nSamples > bufferSize) //Shouldn't happen, and if it happens it'll be slow, but better this than crashing. Will be reset on file close and reset. - { - std::cerr << "Write buffer overrun, resizing to" << nSamples << std::endl; - bufferSize = nSamples; - scaledBuffer.malloc(nSamples); - intBuffer.malloc(nSamples); - } - - double multFactor = 1 / (float(0x7fff) * channel->getChannelBitVolts(0)); - FloatVectorOperations::copyWithMultiply(scaledBuffer.getData(), event->getDataPointer(), multFactor, nSamples); - AudioDataConverters::convertFloatToInt16LE(scaledBuffer.getData(), intBuffer.getData(), nSamples); - - double timestampSec = event->getTimestampInSeconds(); - - CHECK_ERROR(spikeDataSets[electrodeId]->baseDataSet->writeDataBlock(1, BaseDataType::I16, intBuffer)); - CHECK_ERROR(spikeDataSets[electrodeId]->timestampDataSet->writeDataBlock(1, BaseDataType::F64, ×tampSec)); - writeEventMetadata(spikeDataSets[electrodeId], channel, event); - - const int64 sampleNumber = event->getSampleNumber(); - - CHECK_ERROR(spikeDataSets[electrodeId]->sampleNumberDataSet->writeDataBlock(1, BaseDataType::I64, &sampleNumber)); - - - spikeDataSets[electrodeId]->numSamples += 1; - - } - - void NWBFile::writeEvent(int eventID, const EventChannel* channel, const Event* event) - { - - const void* dataSrc; - BaseDataType type; - int8 ttlVal; - String text; - - switch (event->getEventType()) - { - case EventChannel::TTL: - ttlVal = (static_cast(event)->getState() ? 1 : -1) * (static_cast(event)->getLine() + 1); - dataSrc = &ttlVal; - type = BaseDataType::I8; - break; - case EventChannel::TEXT: - text = static_cast(event)->getText(); - dataSrc = text.toUTF8().getAddress(); - type = BaseDataType::STR(text.length()); - break; - default: - dataSrc = static_cast(event)->getBinaryDataPointer(); - type = getEventH5Type(event->getEventType()); - break; - } - - if (eventID == eventDataSets.size()) //MessageCenter event - { - CHECK_ERROR(messagesDataSet->baseDataSet->writeDataBlock(1, BaseDataType::STR(text.length()), text.toUTF8())); - - const int64 sampleNumber = event->getSampleNumber(); - - CHECK_ERROR(messagesDataSet->sampleNumberDataSet->writeDataBlock(1, BaseDataType::I64, &sampleNumber)); - - const double timeSec = event->getTimestampInSeconds(); - - CHECK_ERROR(messagesDataSet->timestampDataSet->writeDataBlock(1, BaseDataType::F64, &timeSec)); - - messagesDataSet->numSamples += 1; - - } - else if (eventDataSets[eventID]) - { - CHECK_ERROR(eventDataSets[eventID]->baseDataSet->writeDataBlock(1, type, dataSrc)); - - const double timeSec = event->getTimestampInSeconds(); - - CHECK_ERROR(eventDataSets[eventID]->timestampDataSet->writeDataBlock(1, BaseDataType::F64, &timeSec)); - - const int64 sampleNumber = event->getSampleNumber(); - - CHECK_ERROR(eventDataSets[eventID]->sampleNumberDataSet->writeDataBlock(1, BaseDataType::I64, &sampleNumber)); - - if (event->getEventType() == EventChannel::TTL) - { - const uint64 ttlWord = static_cast(event)->getWord(); - CHECK_ERROR(eventDataSets[eventID]->ttlWordDataSet->writeDataBlock(1, BaseDataType::U64, &ttlWord)); - } - - eventDataSets[eventID]->numSamples += 1; - } - else - { - //Attempted to write an event to disk from unknown event source - } - - } - - void NWBFile::writeTimestampSyncText(uint16 sourceID, int64 sampleNumber, float sourceSampleRate, String text) - { - CHECK_ERROR(syncMsgDataSet->baseDataSet->writeDataBlock(1, BaseDataType::STR(text.length()), text.toUTF8())); - - CHECK_ERROR(syncMsgDataSet->sampleNumberDataSet->writeDataBlock(1, BaseDataType::I64, &sampleNumber)); - - double timestamp = (double)sampleNumber; - - CHECK_ERROR(syncMsgDataSet->timestampDataSet->writeDataBlock(1, BaseDataType::F64, ×tamp)); - - syncMsgDataSet->numSamples += 1; - } - - - String NWBFile::getFileName() - { - return filename; - } - - bool NWBFile::createTimeSeriesBase(TimeSeries* timeSeries) - { - if (createGroup(timeSeries->basePath)) return false; - CHECK_ERROR(setAttributeStr(" ", timeSeries->basePath, "comments")); - CHECK_ERROR(setAttributeStr(timeSeries->description, timeSeries->basePath, "description")); - CHECK_ERROR(setAttributeStr("core", timeSeries->basePath, "namespace")); - CHECK_ERROR(setAttributeStr(generateUuid(), timeSeries->basePath, "object_id")); - CHECK_ERROR(setAttributeStr(timeSeries->getNeurodataType(), timeSeries->basePath, "neurodata_type")); - return true; - } - - void NWBFile::createDataAttributes(String basePath, float conversion, float resolution, String unit) - { - CHECK_ERROR(setAttribute(BaseDataType::F32, &conversion, basePath + "/data", "conversion")); - CHECK_ERROR(setAttribute(BaseDataType::F32, &resolution, basePath + "/data", "resolution")); - CHECK_ERROR(setAttributeStr(unit, basePath + "/data", "unit")); - } - - HDF5RecordingData* NWBFile::createTimestampDataSet(String path, int chunk_size, float interval) - { - - HDF5RecordingData* tsSet = createDataSet(BaseDataType::F64, 0, chunk_size, path); - - if (!tsSet) - std::cerr << "Error creating timestamp dataset in " << path << std::endl; - else - { - CHECK_ERROR(setAttribute(BaseDataType::F32, &interval, path, "interval")); - CHECK_ERROR(setAttributeStr("seconds", path, "unit")); - } - - return tsSet; - } - - HDF5RecordingData* NWBFile::createSampleNumberDataSet(String path, int chunk_size) - { - HDF5RecordingData* tsSet = createDataSet(BaseDataType::I64, 0, chunk_size, path); - if (!tsSet) - std::cerr << "Error creating sample number dataset in " << path << std::endl; - else - { - const int32 one = 1; - CHECK_ERROR(setAttribute(BaseDataType::I32, &one, path, "interval")); - CHECK_ERROR(setAttributeStr("samples", path, "unit")); - } - return tsSet; - } - -HDF5RecordingData *NWBFile::createChannelConversionDataSet(String path, String description, int chunk_size) -{ - HDF5RecordingData *elSet = createDataSet(BaseDataType::F32, 1, chunk_size, path); - - if (!elSet) - std::cerr << "Error creating electrode dataset in " << path << std::endl; - else - { - CHECK_ERROR(setAttributeStr(description, path, "description")); - CHECK_ERROR(setAttributeStr("hdmf-common", path, "namespace")); - CHECK_ERROR(setAttributeStr(generateUuid(), path, "object_id")); - } - return elSet; -} - -HDF5RecordingData *NWBFile::createElectrodeDataSet(String path, String description, int chunk_size) -{ - HDF5RecordingData *elSet = createDataSet(BaseDataType::I32, 1, chunk_size, path); - if (!elSet) - std::cerr << "Error creating electrode dataset in " << path << std::endl; - else - { - CHECK_ERROR(setAttributeStr(description, path, "description")); - CHECK_ERROR(setAttributeStr("hdmf-common", path, "namespace")); - CHECK_ERROR(setAttributeStr("DynamicTableRegion", path, "neurodata_type")); - CHECK_ERROR(setAttributeStr(generateUuid(), path, "object_id")); - CHECK_ERROR(setAttributeRef("general/extracellular_ephys/electrodes", path, "table")); - } - return elSet; -} - - bool NWBFile::createExtraInfo(String basePath, String name, String desc, String id, uint16 index, uint16 typeIndex) - { - if (createGroup(basePath)) return false; - CHECK_ERROR(setAttributeStr("openephys:/", basePath, "schema_id")); - CHECK_ERROR(setAttributeStr(name, basePath, "name")); - CHECK_ERROR(setAttributeStr(desc, basePath, "description")); - CHECK_ERROR(setAttributeStr(id, basePath, "identifier")); - CHECK_ERROR(setAttribute(BaseDataType::U16, &index, basePath, "source_index")); - CHECK_ERROR(setAttribute(BaseDataType::U16, &typeIndex, basePath, "source_type_index")); - return true; - } - - bool NWBFile::createChannelMetadataSets(String basePath, const MetadataObject* info) - { - if (!info) return false; - if (createGroup(basePath)) return false; - CHECK_ERROR(setAttributeStr("openephys:/", basePath, "schema_id")); - int nMetadata = info->getMetadataCount(); - - for (int i - = 0; i < nMetadata; i++) - { - const MetadataDescriptor* desc = info->getMetadataDescriptor(i); - String fieldName = "Field_" + String(i+1); - String name = desc->getName(); - String description = desc->getDescription(); - String identifier = desc->getIdentifier(); - BaseDataType type = getMetadataH5Type(desc->getType(), desc->getLength()); //only string types use length, for others is always set to 1. If array types are implemented, change this - int length = desc->getType() == MetadataDescriptor::CHAR ? 1 : desc->getLength(); //strings are a single element of length set in the type (see above) while other elements are saved a - HeapBlock data(desc->getDataSize()); - info->getMetadataValue(i)->getValue(static_cast(data.getData())); - createBinaryDataSet(basePath, fieldName, type, length, data.getData()); - String fullPath = basePath + "/" + fieldName; - CHECK_ERROR(setAttributeStr("openephys:/", fullPath, "schema_id")); - CHECK_ERROR(setAttributeStr(name, fullPath, "name")); - CHECK_ERROR(setAttributeStr(description, fullPath, "description")); - CHECK_ERROR(setAttributeStr(identifier, fullPath, "identifier")); - } - return true; - } - - - bool NWBFile::createEventMetadataSets(String basePath, TimeSeries* timeSeries, const MetadataEventObject* info) - { - if (!info) return false; - if (createGroup(basePath)) return false; - CHECK_ERROR(setAttributeStr("openephys:/", basePath, "schema_id")); - int nMetadata = info->getEventMetadataCount(); - - timeSeries->metaDataSet.clear(); //just in case - for (int i = 0; i < nMetadata; i++) - { - const MetadataDescriptor* desc = info->getEventMetadataDescriptor(i); - String fieldName = "Field_" + String(i+1); - String name = desc->getName(); - String description = desc->getDescription(); - String identifier = desc->getIdentifier(); - BaseDataType type = getMetadataH5Type(desc->getType(), desc->getLength()); //only string types use length, for others is always set to 1. If array types are implemented, change this - int length = desc->getType() == MetadataDescriptor::CHAR ? 1 : desc->getLength(); //strings are a single element of length set in the type (see above) while other elements are saved as arrays - String fullPath = basePath + "/" + fieldName; - HDF5RecordingData* dSet = createDataSet(type, 0, length, EVENT_CHUNK_SIZE, fullPath); - if (!dSet) return false; - timeSeries->metaDataSet.add(dSet); - - CHECK_ERROR(setAttributeStr("openephys:/", fullPath, "schema_id")); - CHECK_ERROR(setAttributeStr(name, fullPath, "name")); - CHECK_ERROR(setAttributeStr(description, fullPath, "description")); - CHECK_ERROR(setAttributeStr(identifier, fullPath, "identifier")); - } - return true; - } - - void NWBFile::writeEventMetadata(TimeSeries* timeSeries, const MetadataEventObject* info, const MetadataEvent* event) - { - jassert(timeSeries->metaDataSet.size() == event->getMetadataValueCount()); - jassert(info->getEventMetadataCount() == event->getMetadataValueCount()); - int nMetadata = event->getMetadataValueCount(); - for (int i = 0; i < nMetadata; i++) - { - BaseDataType type = getMetadataH5Type(info->getEventMetadataDescriptor(i)->getType(), info->getEventMetadataDescriptor(i)->getLength()); - timeSeries->metaDataSet[i]->writeDataBlock(1, type, event->getMetadataValue(i)->getRawValuePointer()); - } - - } - - void NWBFile::createTextDataSet(String path, String name, String text) - { - ScopedPointer dSet; - - if (text.isEmpty()) text = " "; //to avoid 0-length strings, which cause errors - BaseDataType type = BaseDataType::STR(text.length()); - - dSet = createDataSet(type, 1, 0, path + "/" + name); - if (!dSet) return; - dSet->writeDataBlock(1, type, text.toUTF8()); - } - - void NWBFile::createBinaryDataSet(String path, String name, BaseDataType type, int length, void* data) - { - ScopedPointer dSet; - if ((length < 1) || !data) return; - - dSet = createDataSet(type, 1, length, 1, path + "/" + name); - if (!dSet) return; - dSet->writeDataBlock(1, type, data); - } - -String NWBFile::generateUuid() -{ - Uuid id; - return id.toDashedString(); -} - - //These two methods whould be easy to adapt to support array types for all base types, for now - //length is only used for string types. - NWBFile::BaseDataType NWBFile::getEventH5Type(EventChannel::Type type, int length) - { - switch (type) - { - case EventChannel::INT8_ARRAY: - return BaseDataType::I8; - case EventChannel::UINT8_ARRAY: - return BaseDataType::U8; - case EventChannel::INT16_ARRAY: - return BaseDataType::I16; - case EventChannel::UINT16_ARRAY: - return BaseDataType::U16; - case EventChannel::INT32_ARRAY: - return BaseDataType::I32; - case EventChannel::UINT32_ARRAY: - return BaseDataType::U32; - case EventChannel::INT64_ARRAY: - return BaseDataType::I64; - case EventChannel::UINT64_ARRAY: - return BaseDataType::U64; - case EventChannel::FLOAT_ARRAY: - return BaseDataType::F32; - case EventChannel::DOUBLE_ARRAY: - return BaseDataType::F64; - case EventChannel::TEXT: - return BaseDataType::STR(length); - default: - return BaseDataType::I8; - } - } - NWBFile::BaseDataType NWBFile::getMetadataH5Type(MetadataDescriptor::MetadataType type, int length) - { - switch (type) - { - case MetadataDescriptor::INT8: - return BaseDataType::I8; - case MetadataDescriptor::UINT8: - return BaseDataType::U8; - case MetadataDescriptor::INT16: - return BaseDataType::I16; - case MetadataDescriptor::UINT16: - return BaseDataType::U16; - case MetadataDescriptor::INT32: - return BaseDataType::I32; - case MetadataDescriptor::UINT32: - return BaseDataType::U32; - case MetadataDescriptor::INT64: - return BaseDataType::I64; - case MetadataDescriptor::UINT64: - return BaseDataType::U64; - case MetadataDescriptor::FLOAT: - return BaseDataType::F32; - case MetadataDescriptor::DOUBLE: - return BaseDataType::F64; - case MetadataDescriptor::CHAR: - return BaseDataType::STR(length); - default: - return BaseDataType::I8; - } - } diff --git a/Source/RecordEngine/NWBFormat.h b/Source/RecordEngine/NWBFormat.h deleted file mode 100644 index e85f6d8..0000000 --- a/Source/RecordEngine/NWBFormat.h +++ /dev/null @@ -1,274 +0,0 @@ -/* - ------------------------------------------------------------------ - - This file is part of the Open Ephys GUI - Copyright (C) 2014 Open Ephys - - ------------------------------------------------------------------ - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . - - */ - -#ifndef NWBFORMAT_H -#define NWBFORMAT_H - -#include -#include -#include - -using namespace OpenEphysHDF5; - -namespace NWBRecording -{ - - typedef Array ContinuousGroup; - - /** - Represents a generic NWB TimeSeries dataset - */ - class TimeSeries - { - public: - - /** Constructor */ - TimeSeries(String rootPath, String name, String description); - - /** Holds the sample data */ - ScopedPointer baseDataSet; - - /** Holds the timestamps (in seconds) for each sample */ - ScopedPointer timestampDataSet; - - /** Holds the sample number for each sample (relative to the start of acquisition) */ - ScopedPointer sampleNumberDataSet; - - /** Holds metadata for this time series */ - OwnedArray metaDataSet; - - /** The path to this dataset within the NWB file */ - String basePath; - - /** The description of this dataset*/ - String description; - - /** Total number of samples written */ - uint64 numSamples = 0; - - /** Get neurodata_type */ - virtual String getNeurodataType() { return "TimeSeries";} - - }; - - namespace ecephys { - /** - Represents an NWB ElectricalSeries dataset - */ - class ElectricalSeries : public TimeSeries - { - public: - /** Constructor */ - ElectricalSeries(String rootPath, String name, String description, - int channel_count, Array channel_conversion); - - /** Holds the sample number for each sample (relative to the start of acquisition) */ - ScopedPointer channelConversionDataSet; - - /** Holds the DynamicTableRegion index of each electrode */ - ScopedPointer electrodeDataSet; - - /** Channel conversion values */ - Array channel_conversion; - - /** Number of channels to write */ - int channel_count; - - /** Get neurodata_type */ - virtual String getNeurodataType() override { return "ElectricalSeries";} - }; - - /** - Represents a sequence of spike events - */ - class SpikeEventSeries : public ElectricalSeries - { - - public: - /** Constructor */ - SpikeEventSeries(String rootPath, String name, String description, - int channel_count, Array channel_conversion); - - /** Get neurodata_type */ - virtual String getNeurodataType() override { return "SpikeEventSeries";} - }; - } - - /** - Represents a TTL event series (not a core NWB data type) - */ - class TTLEventSeries : public TimeSeries - { - public: - /** Constructor */ - TTLEventSeries(String rootPath, String name, String description); - - /** Holds the TTL word for each sample */ - ScopedPointer ttlWordDataSet; - - /** Get neurodata_type */ - virtual String getNeurodataType() override { return "TimeSeries";} - }; - - /** - Represents a sequence of string annotations - */ - class AnnotationSeries : public TimeSeries - { - public: - /** Constructor */ - AnnotationSeries(String rootPath, String name, String description); - - /** Get neurodata_type */ - virtual String getNeurodataType() override { return "AnnotationSeries";} - }; - - /** - - Represents an NWB 2.0 File (a specific type of HDF5 file) - - */ - class NWBFile : public HDF5FileBase - { - public: - - /** Constructor */ - NWBFile(String fName, String ver, String identifier); - - /** Destructor */ - ~NWBFile(); - - /** Creates the groups required for a new recording, given an array of continuous channels, event channels, and spike channels*/ - bool startNewRecording(int recordingNumber, - const Array& continuousArray, - const Array& continuousChannels, - const Array& eventArray, - const Array& electrodeArray); - - /** Writes the num_samples value and closes the relevent datasets */ - void stopRecording(); - - /** Writes continuous data for a particular channel */ - void writeData(int datasetID, int channel, int nSamples, const float* data, float bitVolts); - - /** Writes synchronized timestamps for a particular continuous dataset */ - void writeTimestamps(int datasetID, int nSamples, const double* data); - - /** Writes sample numbers for a particular continuous dataset */ - void writeSampleNumbers(int datasetID, int nSamples, const int64* data); - - /** Writes electrode numbers for a continuous dataset */ - void writeElectrodes(ecephys::ElectricalSeries* electricalSeries, Array electrodeInds); - - /** Writes channel conversion values */ - void writeChannelConversions(ecephys::ElectricalSeries* series); - - /** Writes a spike event*/ - void writeSpike(int electrodeId, const SpikeChannel* channel, const Spike* event); - - /** Writes an event (TEXT or TTL) */ - void writeEvent(int eventID, const EventChannel* channel, const Event* event); - - /** Writes a timestamp sync text event */ - void writeTimestampSyncText(uint16 sourceID, - int64 timestamp, - float sourceSampleRate, - String text); - - /** Returns the name of this NWB file */ - String getFileName() override; - - /** Generate a new uuid string*/ - String generateUuid(); - - protected: - - /** Initializes the default groups */ - int createFileStructure() override; - - private: - - /** Creates a new dataset to hold text data (messages) */ - void createTextDataSet(String path, String name, String text); - - /** Creates a new dataset to hold binary events */ - void createBinaryDataSet(String path, String name, HDF5FileBase::BaseDataType type, int length, void* data); - - /** Returns the HDF5 data type for a given event channel type */ - static HDF5FileBase::BaseDataType getEventH5Type(EventChannel::Type type, int length = 1); - - /** Returns the HDF5 data type for a given metadata type*/ - static HDF5FileBase::BaseDataType getMetadataH5Type(MetadataDescriptor::MetadataType type, int length = 1); - - /** Creates a time series dataset*/ - bool createTimeSeriesBase(TimeSeries* timeSeries); - - /** Creates dataset attributes */ - bool createExtraInfo(String basePath, String name, String desc, String id, uint16 index, uint16 typeIndex); - - /** Creates a dataset of synchronized timestamps (interval = 1/sample_rate) */ - HDF5RecordingData* createTimestampDataSet(String basePath, int chunk_size, float interval); - - /** Creates a dataset of sample numbers */ - HDF5RecordingData* createSampleNumberDataSet(String basePath, int chunk_size); - - /** Creates a dataset for electrode indices */ - HDF5RecordingData* createElectrodeDataSet(String basePath, String description, int chunk_size); - - /** Creates a dataset for electrode indices */ - HDF5RecordingData* createChannelConversionDataSet(String basePath, String description, int chunk_size); - - /** Adds attributes (e.g. conversion, resolution) to a continuous dataset */ - void createDataAttributes(String basePath, float conversion, float resolution, String unit); - - /** Creates a dataset for channel metdata */ - bool createChannelMetadataSets(String basePath, const MetadataObject* info); - - /** Creates a dataset for event metdata */ - bool createEventMetadataSets(String basePath, TimeSeries* timeSeries, const MetadataEventObject* info); - - /** Writes metadata associated with an event*/ - void writeEventMetadata(TimeSeries* timeSeries, const MetadataEventObject* info, const MetadataEvent* event); - - const String filename; - const String GUIVersion; - - OwnedArray continuousDataSets; - OwnedArray spikeDataSets; - OwnedArray eventDataSets; - std::unique_ptr messagesDataSet; - std::unique_ptr syncMsgDataSet; - - const String identifierText; - - HeapBlock scaledBuffer; - HeapBlock intBuffer; - size_t bufferSize; - - JUCE_DECLARE_NON_COPYABLE_WITH_LEAK_DETECTOR(NWBFile); - - }; - -} - -#endif diff --git a/Source/RecordEngine/NWBRecording.cpp b/Source/RecordEngine/NWBRecording.cpp index fa603bf..a38c7c8 100644 --- a/Source/RecordEngine/NWBRecording.cpp +++ b/Source/RecordEngine/NWBRecording.cpp @@ -21,7 +21,12 @@ */ - #include "NWBRecording.h" +#include +#include + +#include "NWBRecording.h" +#include "Channel.hpp" +#include "Utils.hpp" #include "../../plugin-GUI/Source/Processors/RecordNode/RecordNode.h" @@ -32,23 +37,25 @@ NWBRecordEngine::NWBRecordEngine() { - smpBuffer.malloc(MAX_BUFFER_SIZE); } - NWBRecordEngine::~NWBRecordEngine() - { - if (nwb != nullptr) - { - spikeChannels.clear(); - eventChannels.clear(); - continuousChannelGroups.clear(); - datasetIndexes.clear(); - writeChannelIndexes.clear(); - - nwb->close(); - nwb.reset(); - } + { + if (this->nwbfile != nullptr) + { + + this->continuousChannels.clear(); + this->continuousChannelGroups.clear(); + this->spikeChannels.clear(); + + this->recordingArrays.clear(); + this->spikeRecordingArrays.clear(); + this->esContainerIndexes.clear(); + this->spikeContainerIndexes.clear(); + + this->nwbfile->finalize(); + this->nwbfile.reset(); + } } RecordEngineManager* NWBRecordEngine::getEngineManager() @@ -59,161 +66,227 @@ RecordEngineManager* NWBRecordEngine::getEngineManager() param = new EngineParameter(EngineParameter::STR, 0, "Identifier Text", String()); man->addParameter(param); return man; - } - - void NWBRecordEngine::openFiles(File rootFolder, int experimentNumber, int recordingNumber) - { - - if (recordingNumber == 0) // new file needed - { - - spikeChannels.clear(); - eventChannels.clear(); - continuousChannels.clear(); - continuousChannelGroups.clear(); - datasetIndexes.clear(); - writeChannelIndexes.clear(); - - // New file for each experiment, e.g. experiment1.nwb, epxperiment2.nwb, etc. - String basepath = rootFolder.getFullPathName() + - rootFolder.getSeparatorString() + - "experiment" + String(experimentNumber) + - ".nwb"; - - if (nwb != nullptr) - { - nwb->close(); - nwb.reset(); - } - - // create a unique identifier for the file if it doesn't exist - Uuid identifier; - identifierText = identifier.toString(); - - nwb = std::make_unique(basepath, CoreServices::getGUIVersion(), identifierText); - - // get pointers to all continuous channels for electrode table - for (int i = 0; i < recordNode->getNumOutputs(); i++) - { - const ContinuousChannel* channelInfo = getContinuousChannel(i); // channel info object - - continuousChannels.add(channelInfo); - } - - datasetIndexes.insertMultiple(0, 0, getNumRecordedContinuousChannels()); - writeChannelIndexes.insertMultiple(0, 0, getNumRecordedContinuousChannels()); - continuousChannelGroups.clear(); - - int streamIndex = -1; - uint16 lastStreamId = 0; - int indexWithinStream = 0; - for (int ch = 0; ch < getNumRecordedContinuousChannels(); ch++) - { - int globalIndex = getGlobalIndex(ch); // the global channel index (across all channels entering the Record Node) - int localIndex = getLocalIndex(ch); // the local channel index (within a stream) - - const ContinuousChannel* channelInfo = getContinuousChannel(globalIndex); // channel info object - - int sourceId = channelInfo->getSourceNodeId(); - int streamId = channelInfo->getStreamId(); - - if (streamId != lastStreamId) - { - streamIndex++; - indexWithinStream = 0; + void NWBRecordEngine::openFiles(File rootFolder, int experimentNumber, int recordingNumber) + { + if (recordingNumber == 0) // new file needed + { + // clear any existing data and nwbfile + this->continuousChannels.clear(); + this->continuousChannelGroups.clear(); + this->spikeChannels.clear(); + + this->recordingArrays.clear(); + this->spikeRecordingArrays.clear(); + this->esContainerIndexes.clear(); + this->spikeContainerIndexes.clear(); + + if (this->nwbfile != nullptr) + { + this->nwbfile->finalize(); + this->nwbfile.reset(); + } - ContinuousGroup newGroup; - continuousChannelGroups.add(newGroup); + // create the io object + char separator = std::filesystem::path::preferred_separator; + std::string separatorStr(1, separator); // Convert char to std::string + std::string filename = rootFolder.getFullPathName().toStdString() + separatorStr + + "experiment_aqnwb" + std::to_string(experimentNumber) + ".nwb"; - } + this->io = AQNWB::createIO("HDF5", filename); - continuousChannelGroups.getReference(streamIndex).add(channelInfo); + // create recording array mapping for channel information + NWBRecordEngine::createRecordingArrays(); - datasetIndexes.set(ch, streamIndex); - writeChannelIndexes.set(ch, indexWithinStream++); + // create the nwbfile + std::string dataCollection = "Open Ephys GUI Version " + CoreServices::getGUIVersion().toStdString(); + this->nwbfile = std::make_unique(AQNWB::generateUuid(), io); + this->nwbfile->initialize("Recording with the Open Ephys GUI", dataCollection); + // TODO - have option to initialize cache size based on # of channels - lastStreamId = streamId; - } + // create recording containers + this->recordingContainers = std::make_unique(); + this->nwbfile->createElectricalSeries( + this->recordingArrays, this->recordingArraysNames, AQNWB::BaseDataType::I16, this->recordingContainers.get(), this->esContainerIndexes); + // TODO add io_settings to set chunk size for different data types - for (int i = 0; i < getNumRecordedEventChannels(); i++) - eventChannels.add(getEventChannel(i)); + this->nwbfile->createSpikeEventSeries( + this->spikeRecordingArrays, this->spikeRecordingArraysNames, AQNWB::BaseDataType::I16, this->recordingContainers.get(), this->spikeContainerIndexes); - for (int i = 0; i < getNumRecordedSpikeChannels(); i++) - spikeChannels.add(getSpikeChannel(i)); + // start recording + this->io->startRecording(); + } + } - //open the file - nwb->open(getNumRecordedContinuousChannels() + continuousChannelGroups.size() + eventChannels.size() + spikeChannels.size()); //total channels + timestamp arrays, to create a big enough buffer - //create the recording - nwb->startNewRecording(recordingNumber, continuousChannelGroups, continuousChannels, eventChannels, spikeChannels); - } - } - void NWBRecordEngine::closeFiles() { - nwb->stopRecording(); + this->io->stopRecording(); + this->nwbfile->finalize(); } - - void NWBRecordEngine::writeContinuousData(int writeChannel, int realChannel, const float* dataBuffer, const double* timestampBuffer, int size) -{ - nwb->writeData(datasetIndexes[writeChannel], - writeChannelIndexes[writeChannel], - size, - dataBuffer, - getContinuousChannel(realChannel)->getBitVolts()); - - /* All channels in a dataset have the same number of samples and share timestamps. - But since this method is called asynchronously, the timestamps might not be - in sync during acquisition, so we chose a channel and write the timestamps - when writing that channel's data */ - if (writeChannelIndexes[writeChannel] == 0) - { - int64 baseTS = getLatestSampleNumber(writeChannel); - - for (int i = 0; i < size; i++) - { - smpBuffer[i] = baseTS + i; +{ + // get channel info - add this to RecordingArray or ChannelVector when we make it a class + AQNWB::Channel* channel = nullptr; + AQNWB::Types::SizeType datasetIndex = 0; + for (auto& channelVector : this->recordingArrays) { + for (auto& ch : channelVector) { + if (ch.globalIndex == realChannel) { + channel = &ch; + break; + } } - - nwb->writeTimestamps(datasetIndexes[writeChannel], size, timestampBuffer); - nwb->writeSampleNumbers(datasetIndexes[writeChannel], size, smpBuffer); } + + std::unique_ptr intBuffer = AQNWB::transformToInt16(static_cast(size), channel->getBitVolts(), dataBuffer); + this->recordingContainers->writeElectricalSeriesData(this->esContainerIndexes[channel->groupIndex], + *channel, + static_cast(size), + intBuffer.get(), + timestampBuffer); + + // TODO - save sample numbers as well for offline syncing } void NWBRecordEngine::writeEvent(int eventIndex, const MidiMessage& event) -{ - const EventChannel* channel = getEventChannel(eventIndex); - EventPtr eventStruct = Event::deserialize(event, channel); +{ + // TODO - replace with AQNWB + // const EventChannel* channel = getEventChannel(eventIndex); + // EventPtr eventStruct = Event::deserialize(event, channel); - nwb->writeEvent(eventIndex, channel, eventStruct); + // nwb->writeEvent(eventIndex, channel, eventStruct); } void NWBRecordEngine::writeTimestampSyncText(uint64 streamId, int64 timestamp, float sourceSampleRate, String text) { - nwb->writeTimestampSyncText(streamId, timestamp, sourceSampleRate, text); + // TODO - replacew with AQNWB + // nwb->writeTimestampSyncText(streamId, timestamp, sourceSampleRate, text); } void NWBRecordEngine::writeSpike(int electrodeIndex, const Spike* spike) { - const SpikeChannel* channel = getSpikeChannel(electrodeIndex); - - nwb->writeSpike(electrodeIndex, channel, spike); + // extract info from spike channel + const SpikeChannel* spikeChannel = getSpikeChannel(electrodeIndex); + SizeType numSamplesPerChannel = static_cast(spikeChannel->getTotalSamples()); + SizeType numChannels = static_cast(spikeChannel->getNumChannels()); + SizeType numSamples = numSamplesPerChannel * numChannels; + + // extract info from spike object + double timestamps = spike->getTimestampInSeconds(); + std::unique_ptr intData = AQNWB::transformToInt16(static_cast(numSamples), + spikeChannel->getSourceChannels()[0]->getBitVolts(), + spike->getDataPointer()); + + // write spike data + this->recordingContainers->writeSpikeEventData(this->spikeContainerIndexes[electrodeIndex], + numSamplesPerChannel, + numChannels, + intData.get(), + ×tamps); + + // TODO - add writeEventMetadata functionalities } - - void NWBRecordEngine::setParameter(EngineParameter& parameter) { strParameter(0, identifierText); } + +void NWBRecordEngine::createRecordingArrays() +{ + // get pointers to all continuous channels for electrode table + for (int i = 0; i < recordNode->getNumOutputs(); i++) + { + const ContinuousChannel* channelInfo = getContinuousChannel(i); // channel info object + this->continuousChannels.add(channelInfo); + } + + // add continuous channels + int streamIndex = -1; + uint16 lastStreamId = 0; + for (int ch = 0; ch < getNumRecordedContinuousChannels(); ch++) + { + int globalIndex = getGlobalIndex(ch); // the global channel index (across all channels entering the Record Node) + int localIndex = getLocalIndex(ch); // the local channel index (within a stream) + + const ContinuousChannel* channelInfo = getContinuousChannel(globalIndex); // channel info object + if (channelInfo->getStreamId() != lastStreamId) + { + streamIndex++; + ContinuousGroup newGroup; + this->continuousChannelGroups.add(newGroup); + } + + this->continuousChannelGroups.getReference(streamIndex).add(channelInfo); + lastStreamId = channelInfo->getStreamId(); + } + + // add spike channels + for (int i = 0; i < getNumRecordedSpikeChannels(); i++) { + spikeChannels.add(getSpikeChannel(i)); + } + + // create recording arrays for continuous groups in nwb file + for (int streamIndex = 0; streamIndex < this->continuousChannelGroups.size(); streamIndex++) + { + std::vector channelVector; + + for (auto& channelInfo : this->continuousChannelGroups[streamIndex]) + { + std::string name = channelInfo->getName().toStdString(); + std::string groupName = channelInfo->getSourceNodeName().toStdString() + "-" + + std::to_string(channelInfo->getSourceNodeId()) + + "." + channelInfo->getStreamName().toStdString(); + + channelVector.push_back(AQNWB::Channel(name, + groupName, + streamIndex, + channelInfo->getLocalIndex(), + channelInfo->getGlobalIndex(), + 1e6, + channelInfo->getSampleRate(), + channelInfo->getBitVolts())); + } + this->recordingArrays.push_back(channelVector); + this->recordingArraysNames.push_back(channelVector[0].groupName); + } + + // create recording arrays for spike channels in nwb file + for (int i = 0; i < this->spikeChannels.size(); i++) + { + std::vector channelVector; + + const SpikeChannel* spikeChannel = this->spikeChannels[i]; + std::string sourceName = spikeChannel->getSourceNodeName().toStdString() + "-" + + std::to_string(spikeChannel->getSourceNodeId()) + + "." + spikeChannel->getStreamName().toStdString() + '.' + spikeChannel->getName().toStdString(); + + for (int ch = 0; ch < spikeChannel->getNumChannels(); ch++) + { + const ContinuousChannel* schan = spikeChannel->getSourceChannels()[ch]; + std::string continuousSourceName = schan->getSourceNodeName().toStdString() + "-" + + std::to_string(schan->getSourceNodeId()) + + "." + schan->getStreamName().toStdString(); + AQNWB::Channel channel(schan->getName().toStdString(), + continuousSourceName, + i, + schan->getLocalIndex(), + schan->getGlobalIndex(), + 1e6, + schan->getSampleRate(), + schan->getBitVolts()); + channelVector.push_back(channel); + } + this->spikeRecordingArrays.push_back(channelVector); + this->spikeRecordingArraysNames.push_back(sourceName); + } +} diff --git a/Source/RecordEngine/NWBRecording.h b/Source/RecordEngine/NWBRecording.h index a2aba61..c841f6a 100644 --- a/Source/RecordEngine/NWBRecording.h +++ b/Source/RecordEngine/NWBRecording.h @@ -26,7 +26,11 @@ #include -#include "NWBFormat.h" +#include "BaseIO.hpp" +#include "nwb/NWBFile.hpp" +#include "nwb/RecordingContainers.hpp" + +typedef Array ContinuousGroup; namespace NWBRecording { @@ -76,35 +80,48 @@ namespace NWBRecording /** Allows the file identifier to be set externally*/ void setParameter(EngineParameter ¶meter) override; + /** Create recording arrays */ + void createRecordingArrays(); + private: - /** Pointer to the current NWB file */ - std::unique_ptr nwb; + /** NWB file */ + std::unique_ptr nwbfile; + + /** NWB recording container manager */ + std::unique_ptr recordingContainers; + + /** NWB I/O object */ + std::shared_ptr io; + + /** Holds channel information and ids */ + std::vector recordingArrays; + + /** Holds names of the recordingArrays */ + std::vector recordingArraysNames; - /** For each incoming recorded channel, which dataset (stream) is it associated with? */ - Array datasetIndexes; + /** Holds channel information and ids */ + std::vector spikeRecordingArrays; + + /** Holds names of the spikeRecordingArrays */ + std::vector spikeRecordingArraysNames; - /** For each incoming recorded channel, what is the local index within a stream? */ - Array writeChannelIndexes; + /** Holds the indexes of the ElectricalSeries containers added to recordingContainers */ + std::vector esContainerIndexes; + + /** Holds the indexes of the ElectricalSeries containers added to recordingContainers */ + std::vector spikeContainerIndexes; /** Holds pointers to all recorded channels within a stream */ Array continuousChannelGroups; - /** Holds pointers to all recorded event channels*/ - Array eventChannels; - /** Holds pointers to all recorded spike channels*/ Array spikeChannels; /** Holds pointers to all incoming continuous channels (used for electrode table)*/ Array continuousChannels; - /** Holds integer sample numbers for writing */ - HeapBlock smpBuffer; - - /** The identifier for the current file (can be set externally) */ + // /** The identifier for the current file (can be set externally) */ String identifierText; - - JUCE_DECLARE_NON_COPYABLE_WITH_LEAK_DETECTOR(NWBRecordEngine); }; } diff --git a/Source/aqnwb b/Source/aqnwb new file mode 160000 index 0000000..3a12145 --- /dev/null +++ b/Source/aqnwb @@ -0,0 +1 @@ +Subproject commit 3a121450dbb6c0ce1d37327a13c587b4d44072a4