From dfaea2a09fb5d03f13bd69a76373f47153d3d7f5 Mon Sep 17 00:00:00 2001 From: Steph Prince <40640337+stephprince@users.noreply.github.com> Date: Tue, 27 Feb 2024 16:18:47 -0800 Subject: [PATCH] add initial file structure creation --- src/BaseIO.hpp | 27 ++--- src/HDF5IO.cpp | 76 ++++++-------- src/HDF5IO.hpp | 31 +++--- src/NWBFile.cpp | 232 ++++++++++++++++++++++-------------------- src/NWBFile.hpp | 35 +++---- tests/aq-nwb_test.cpp | 91 +++++++---------- 6 files changed, 239 insertions(+), 253 deletions(-) diff --git a/src/BaseIO.hpp b/src/BaseIO.hpp index 9fb51102..e7b0ffb3 100644 --- a/src/BaseIO.hpp +++ b/src/BaseIO.hpp @@ -71,7 +71,10 @@ class BaseIO virtual std::string getFileName() = 0; /** Opens the file for writing */ - virtual int open(std::string fileName) = 0; + virtual int open() = 0; + + /** Opens the file for writing */ + virtual int open(bool newfile) = 0; /** Closes the file */ virtual void close() = 0; @@ -85,23 +88,26 @@ class BaseIO // /** Sets a string attribute at a given location in the file */ virtual int setAttribute(const std::string& data, - std::string path, - std::string name) = 0; + std::string path, + std::string name) = 0; /** Sets a std::string array attribute at a given location in the file */ virtual int setAttribute(const std::vector& data, - std::string path, - std::string name) = 0; + std::string path, + std::string name) = 0; /** Sets a std::string array attribute at a given location in the file */ virtual int setAttribute(const std::vector& data, - std::string path, - std::string name, - size_t maxSize) = 0; + std::string path, + std::string name, + size_t maxSize) = 0; /** Creates a new group */ virtual int createGroup(std::string path) = 0; + /** Creates a non-modifiable dataset with a string value */ + virtual void createStringDataSet(std::string path, std::string value) = 0; + // ------------------------------------------------------------ // OTHER METHODS // ------------------------------------------------------------ @@ -116,16 +122,11 @@ class BaseIO const std::string filename; protected: - /** Creates the basic file structure upon opening */ - virtual int createFileStructure() = 0; // TODO - not sure if this needs to be - // here or ust in the NWB side - /** Creates a new group (ignores if it exists) */ virtual int createGroupIfDoesNotExist(std::string path) = 0; bool readyToOpen; bool opened; - bool newfile; }; /** diff --git a/src/HDF5IO.cpp b/src/HDF5IO.cpp index 34bf3ecc..b1f7d994 100644 --- a/src/HDF5IO.cpp +++ b/src/HDF5IO.cpp @@ -3,10 +3,10 @@ #include #include #include -#include #include "HDF5IO.hpp" +#include using namespace H5; using namespace AQNWBIO; @@ -15,6 +15,11 @@ using namespace AQNWBIO; HDF5IO::HDF5IO() {} +HDF5IO::HDF5IO(std::string fileName) + : filename(fileName) +{ +} + HDF5IO::~HDF5IO() { close(); @@ -25,27 +30,18 @@ std::string HDF5IO::getFileName() return filename; } -int HDF5IO::open(std::string fileName) +int HDF5IO::open() { - filename = fileName; - - if (!readyToOpen) - return -1; - - if (std::filesystem::exists(getFileName())) - newfile = false; - else - newfile = true; - - if (opened) - return -1; - - return open(newfile); + if (std::filesystem::exists(getFileName())) { + return open(false); + } else { + return open(true); + } } int HDF5IO::open(bool newfile) { - int accFlags, ret = 0; + int accFlags = 0; if (opened) return -1; @@ -56,21 +52,12 @@ int HDF5IO::open(bool newfile) accFlags = H5F_ACC_TRUNC; else accFlags = H5F_ACC_RDWR; + file = std::make_unique( getFileName(), accFlags, FileCreatPropList::DEFAULT, props); opened = true; - if (newfile) { - ret = createFileStructure(); - } - - if (ret) { - file = nullptr; - opened = false; - std::cerr << "Error creating file structure" << std::endl; - } - - return ret; + return 0; } void HDF5IO::close() @@ -79,11 +66,6 @@ void HDF5IO::close() opened = false; } -int HDF5IO::createFileStructure() // TODO - move this to NWB class -{ - return 0; -} - int HDF5IO::setAttribute(BaseDataType type, const void* data, std::string path, @@ -132,8 +114,8 @@ int HDF5IO::setAttribute(BaseDataType type, } int HDF5IO::setAttribute(const std::string& data, - std::string path, - std::string name) + std::string path, + std::string name) { std::vector dataPtrs; dataPtrs.push_back(data.c_str()); @@ -142,8 +124,8 @@ int HDF5IO::setAttribute(const std::string& data, } int HDF5IO::setAttribute(const std::vector& data, - std::string path, - std::string name) + std::string path, + std::string name) { std::vector dataPtrs; size_t maxLength = 0; @@ -157,9 +139,9 @@ int HDF5IO::setAttribute(const std::vector& data, } int HDF5IO::setAttribute(const std::vector& data, - std::string path, - std::string name, - size_t maxSize) + std::string path, + std::string name, + size_t maxSize) { H5Object* loc; Group gloc; @@ -251,20 +233,28 @@ HDF5RecordingData* HDF5IO::getDataSet(std::string path) data = std::make_unique(file->openDataSet(path)); return new HDF5RecordingData(data.release()); } catch (DataSetIException error) { - // std::cout << "DataSetIException" << std::endl; error.printErrorStack(); return nullptr; } catch (FileIException error) { - // std::cout << "FileIException" << std::endl; error.printErrorStack(); return nullptr; } catch (DataSpaceIException error) { - // std::cout << "DataSpaceIException" << std::endl; error.printErrorStack(); return nullptr; } } +void HDF5IO::createStringDataSet(std::string path, std::string value) +{ + std::unique_ptr dataset; + DataType H5type = getH5Type(BaseDataType::STR(value.length())); + DataSpace dSpace(H5S_SCALAR); + + dataset = + std::make_unique(file->createDataSet(path, H5type, dSpace)); + dataset->write(value.c_str(), H5type); +} + HDF5RecordingData* HDF5IO::createDataSet(BaseDataType type, int sizeX, int chunkX, diff --git a/src/HDF5IO.hpp b/src/HDF5IO.hpp index cb759487..ad9f6b0d 100644 --- a/src/HDF5IO.hpp +++ b/src/HDF5IO.hpp @@ -24,14 +24,20 @@ class HDF5IO : public BaseIO /** Constructor */ HDF5IO(); + /** Constructor */ + HDF5IO(std::string fileName); + /** Destructor */ ~HDF5IO(); /** Returns the full path to the HDF5 file */ std::string getFileName() override; - /** Opens the file for writing */ - int open(std::string fileName) override; + /** Opens existing file or creates new file for writing */ + int open() override; + + /** Opens existing file or creates new file for writing */ + int open(bool newfile) override; /** Closes the file */ void close() override; @@ -69,12 +75,12 @@ class HDF5IO : public BaseIO /** Creates a new group (throws an exception if it exists) */ int createGroup(std::string path) override; - /** Creates the basic file structure upon opening */ - int createFileStructure() override; - // /** Returns a pointer to a dataset at a given path*/ HDF5RecordingData* getDataSet(std::string path); + /** Creates a non-modifiable dataset with a string value */ + void createStringDataSet(std::string path, std::string value) override; + /** aliases for createDataSet */ HDF5RecordingData* createDataSet(BaseDataType type, int sizeX, @@ -96,21 +102,12 @@ class HDF5IO : public BaseIO int chunkY, std::string path); - inline int showError(const char* error) - { - std::cerr << error << std::endl; - return -1; - } - protected: std::string filename; /** Creates a new group (ignores if it exists) */ int createGroupIfDoesNotExist(std::string path) override; - /** Opens existing file or creates new file */ - int open(bool newfile); - private: std::unique_ptr file; @@ -150,3 +147,9 @@ class HDF5RecordingData : public BaseRecordingData private: std::unique_ptr dSet; }; + +inline int showError(const char* error) +{ + std::cerr << error << std::endl; + return -1; +} \ No newline at end of file diff --git a/src/NWBFile.cpp b/src/NWBFile.cpp index d548ef9c..a95983af 100644 --- a/src/NWBFile.cpp +++ b/src/NWBFile.cpp @@ -1,40 +1,50 @@ #include -#include +#include #include +#include +#include +#include +#include +#include #include "NWBFile.hpp" + +#include +#include +#include + #include "BaseIO.hpp" using namespace AQNWBIO; namespace fs = std::filesystem; -#define MAX_BUFFER_SIZE 40960 - +#define MAX_BUFFER_SIZE \ + 40960 // TODO - maybe instead use constexpr size_t MAX_BUFFER_SIZE = 40960; // NWBFile -NWBFile::NWBFile(std::string fileName, std::string idText, std::unique_ptr io) - : - filename(fileName), - identifierText(idText), - io(std::move(io)) +NWBFile::NWBFile(std::string idText, std::unique_ptr io) + : identifierText(idText) + , io(std::move(io)) { - readyToOpen = true; - -// scaledBuffer.malloc(MAX_BUFFER_SIZE); TODO - JUCE types, need to adapt for std library -// intBuffer.malloc(MAX_BUFFER_SIZE); + // scaledBuffer.malloc(MAX_BUFFER_SIZE); TODO - JUCE types, need to adapt + // for std library intBuffer.malloc(MAX_BUFFER_SIZE); bufferSize = MAX_BUFFER_SIZE; } NWBFile::~NWBFile() {} -void NWBFile::open() +void NWBFile::initialize() { - io->open(filename); - createFileStructure(); + if (std::filesystem::exists(io->getFileName())) { + io->open(false); + } else { + io->open(true); + createFileStructure(); + } } -void NWBFile::close() +void NWBFile::finalize() { io->close(); } @@ -46,70 +56,43 @@ int NWBFile::createFileStructure() io->setAttribute(NWBVersion, "/", "nwb_version"); io->setAttribute(identifierText, "/", "object_id"); - if (io->createGroup("/acquisition")) - return -1; - - if (io->createGroup("/analysis")) - return -1; - - // TODO - use chrono to get iso date time - // std::string time = Time::getCurrentTime().formatted("%Y-%m-%dT%H:%M:%S") - // + Time::getCurrentTime().getUTCOffsetString(true); - - // createTextDataSet("", "file_create_date", time); - - // Get the current time point - auto now = std::chrono::system_clock::now(); - std::time_t currentTime = std::chrono::system_clock::to_time_t(now); - // Convert the time_t to a tm structure - std::tm* timeInfo = std::localtime(¤tTime); - // Format the time in ISO 8601 date-time format - char buffer[20]; // ISO 8601 date-time format requires at least 20 characters - std::strftime(buffer, sizeof(buffer), "%Y-%m-%dT%H:%M:%S", timeInfo); - - if (io->createGroup("/general")) - return -1; - if (io->createGroup("general/devices")) - return -1; - if (io->createGroup("general/extracellular_ephys")) - return -1; - if (io->createGroup("general/extracellular_ephys/electrodes")) - return -1; + io->createGroup("/acquisition"); + io->createGroup("/analysis"); + io->createGroup("/processing"); + io->createGroup("/stimulus"); + io->createGroup("/stimulus/presentation"); + io->createGroup("/stimulus/templates"); + io->createGroup("/general"); + io->createGroup("general/devices"); + io->createGroup("general/extracellular_ephys"); + io->createGroup("general/extracellular_ephys/electrodes"); const std::vector colnames = {"group", "group_name", "location"}; io->setAttribute( colnames, "general/extracellular_ephys/electrodes", "colnames"); io->setAttribute("metadata about extracellular electrodes", - "general/extracellular_ephys/electrodes", - "description"); + "general/extracellular_ephys/electrodes", + "description"); io->setAttribute( "hdmf-common", "general/extracellular_ephys/electrodes", "namespace"); io->setAttribute("DynamicTable", - "general/extracellular_ephys/electrodes", - "neurodata_type"); - // io->setAttributeStr(generateUuid(), - // "general/extracellular_ephys/electrodes", "object_id"); - - if (io->createGroup("/processing")) - return -1; - - if (io->createGroup("/stimulus")) - return -1; - if (io->createGroup("/stimulus/presentation")) - return -1; - if (io->createGroup("/stimulus/templates")) - return -1; - - if (io->createGroup("/specifications")) - return -1; + "general/extracellular_ephys/electrodes", + "neurodata_type"); + io->setAttribute( + generateUuid(), "general/extracellular_ephys/electrodes", "object_id"); + + io->createGroup("/specifications"); cacheSpecifications("core/", NWBVersion); cacheSpecifications("hdmf-common/", HDMFVersion); - // createStringDataSet("/session_description", "Recording with the Open Ephys - // GUI"); createStringDataSet("/session_start_time", time); - // createStringDataSet("/timestamps_reference_time", time); - // createStringDataSet("/identifier", "test-identifier"); + std::string time = getCurrentTime(); + io->createStringDataSet("/file_create_date", + time); // TODO - this should be an array + io->createStringDataSet("/session_description", "a recording session"); + io->createStringDataSet("/session_start_time", time); + io->createStringDataSet("/timestamps_reference_time", time); + io->createStringDataSet("/identifier", "test-identifier"); return 0; } @@ -118,68 +101,91 @@ bool NWBFile::startRecording(int recordingNumber) { // all recorded data is stored in the "acquisition" group std::string rootPath = "/acquisition/"; - + // TODO - tons of setup things here return true; } -void NWBFile::stopRecording() -{ - -} +void NWBFile::stopRecording() {} void NWBFile::writeData( int datasetID, int channel, int nSamples, const float* data, float bitVolts) { - // TODO - some things here + // TODO - some things here } -std::string NWBFile::getFileName() +void NWBFile::cacheSpecifications(std::string specPath, + std::string versionNumber) { - return filename; + io->createGroup("/specifications/" + specPath); + io->createGroup("/specifications/" + specPath + versionNumber); + + fs::path currentFile = __FILE__; + fs::path schemaDir = currentFile.parent_path().parent_path() + / "resources/spec" / specPath / versionNumber; + + for (auto const& entry : fs::directory_iterator {schemaDir}) + if (fs::is_regular_file(entry) && entry.path().extension() == ".json") { + std::string specName = + entry.path().filename().replace_extension("").string(); + if (specName.find("namespace") != std::string::npos) + specName = "namespace"; + + std::ifstream schemaFile(entry.path()); + std::stringstream buffer; + buffer << schemaFile.rdbuf(); + + io->createStringDataSet( + "/specifications/" + specPath + versionNumber + "/" + specName, + buffer.str()); + } } +std::string NWBFile::generateUuid() +{ + boost::uuids::uuid uuid = boost::uuids::random_generator()(); + std::string uuidStr = boost::uuids::to_string(uuid); -void NWBFile::cacheSpecifications(std::string specPath, std::string versionNumber) + return uuidStr; +} + +std::string NWBFile::getCurrentTime() { - io->createGroup("/specifications/" + specPath); - io->createGroup("/specifications/" + specPath + versionNumber); + // Get current time + auto currentTime = + std::chrono::system_clock::to_time_t(std::chrono::system_clock::now()); - // fs::path currentFile = __FILE__; - // fs::path schemaDir = currentFile.parent_path().parent_path().parent_path() / "resources/nwb-schema" / specPath / versionNumber; + // Convert to tm struct to extract date and time components + std::tm utcTime = *std::gmtime(¤tTime); - // for (auto const& entry : fs::directory_iterator{schemaDir}) - // if (fs::is_regular_file(entry) && entry.path().extension() == ".json") - // { - // std::string specName = entry.path().filename().replace_extension("").string(); - // if (specName.contains("namespace")) specName = "namespace"; - // createStringDataSet("/specifications/" + specPath + versionNumber + specName, schemaFile.loadFileAsString()); TODO - replace with actual schema writing - // } -} + // Format the date and time in ISO 8601 format with the UTC offset + std::ostringstream oss; + oss << std::put_time(&utcTime, "%FT%T%z"); + return oss.str(); +} // NWBRecordingEngine - NWBRecordingEngine::NWBRecordingEngine() - { - // smpBuffer.malloc(MAX_BUFFER_SIZE); // TODO - JUCE type, need to adapt - } - - - NWBRecordingEngine::~NWBRecordingEngine() - { - if (nwb != nullptr) - { - // nwb->close(); // TODO - figure out which BaseIO / RecordingData things to handle - // nwb.reset(); - } - } - - void NWBRecordingEngine::openFiles(std::string rootFolder, int experimentNumber, int recordingNumber) - { - } - - - void NWBRecordingEngine::closeFiles() - { - nwb->stopRecording(); - } \ No newline at end of file +NWBRecordingEngine::NWBRecordingEngine() +{ + // smpBuffer.malloc(MAX_BUFFER_SIZE); // TODO - JUCE type, need to adapt +} + +NWBRecordingEngine::~NWBRecordingEngine() +{ + if (nwb != nullptr) { + // nwb->close(); // TODO - figure out which BaseIO / RecordingData things + // to handle nwb.reset(); + } +} + +void NWBRecordingEngine::openFiles(std::string rootFolder, + int experimentNumber, + int recordingNumber) +{ +} + +void NWBRecordingEngine::closeFiles() +{ + nwb->stopRecording(); +} \ No newline at end of file diff --git a/src/NWBFile.hpp b/src/NWBFile.hpp index 286324f3..356baf74 100644 --- a/src/NWBFile.hpp +++ b/src/NWBFile.hpp @@ -1,6 +1,7 @@ #pragma once #include + #include "BaseIO.hpp" #include "HDF5IO.hpp" @@ -15,18 +16,20 @@ class NWBFile { public: /** Constructor */ - NWBFile(std::string fileName, std::string idText, std::unique_ptr io); + NWBFile(std::string idText, std::unique_ptr io); NWBFile(const NWBFile&) = delete; // non construction-copyable NWBFile& operator=(const NWBFile&) = delete; // non copiable /** Destructor */ ~NWBFile(); - /** Opens the file for writing */ - void open(); + /** Initilaizes the file - opens and sets up file structure */ + void initialize(); - void close(); + /** Finalizes the file - closes*/ + void finalize(); + /** Starts a recording */ bool startRecording(int recordingNumber); /** Writes the num_samples value and closes the relevant datasets */ @@ -39,14 +42,8 @@ class NWBFile const float* data, float bitVolts); - /** Saves the specification files for the schema */ - void cacheSpecifications(std::string specPath, std::string versionNumber); - - /** Returns the name of this NWB file */ - std::string getFileName(); - -// /** Generate a new uuid string*/ -// std::string generateUuid(); + /** Generate a new uuid string*/ + std::string generateUuid(); /** Indicate NWB version files will be saved as */ const std::string NWBVersion = "2.7.0"; @@ -59,24 +56,28 @@ class NWBFile int createFileStructure(); private: + /** Saves the specification files for the schema */ + void cacheSpecifications(std::string specPath, std::string versionNumber); + /** Creates a new dataset to hold text data (messages) */ void createTextDataSet(std::string path, std::string name, std::string text); - const std::string filename; + /** Creates a new dataset to hold text data (messages) */ + std::string getCurrentTime(); + const std::string identifierText; std::unique_ptr io; - std::vector scaledBuffer; // TODO - switched out for std::vector, not sure if it's best substitute + std::vector scaledBuffer; // TODO - switched out for std::vector, not + // sure if it's best substitute std::vector intBuffer; size_t bufferSize; - bool readyToOpen; - }; /** Represents an NWB recording engine to manage recording process - */ +*/ class NWBRecordingEngine { public: diff --git a/tests/aq-nwb_test.cpp b/tests/aq-nwb_test.cpp index b9364825..1cc78452 100644 --- a/tests/aq-nwb_test.cpp +++ b/tests/aq-nwb_test.cpp @@ -9,83 +9,68 @@ namespace fs = std::filesystem; - -// TODO - change setup/teardown of data folder when running tests std::string getTestFilePath(std::string filename) { + // create data directory if it doesn't exist fs::path dirPath = fs::current_path() / "data"; fs::directory_entry dir(dirPath); if (!dir.exists()) { fs::create_directory(dir); } + + // get filename and remove old file fs::path filepath = dirPath / filename; + if (fs::exists(filepath)) { + fs::remove(filepath); + } return filepath.u8string(); } - -TEST_CASE("write_attribute", "[io]") +TEST_CASE("write_attributes", "[hdf5io]") { - std::string filename = getTestFilePath("test_attribute.h5"); - - // create file - HDF5IO hdf5io; - hdf5io.open(filename); + // create and open file + std::string filename = getTestFilePath("test_attributes.h5"); + HDF5IO hdf5io(filename); + hdf5io.open(); - // Write data to file - const signed int data = 1; hdf5io.createGroup("/data"); - hdf5io.setAttribute( - AQNWBIO::BaseDataType::I32, &data, "/data", "single_value"); - - // close file - hdf5io.close(); - // std::remove(filename.c_str()); -} - -TEST_CASE("write_int_array", "[io]") -{ - std::string filename = getTestFilePath("test_int_array.h5"); - - HDF5IO hdf5io; - hdf5io.open(filename); - // Setup data structures - const int data[] = {1, 2, 3, 4, 5}; - const int dataSize = sizeof(data) / sizeof(data[0]); - - // Write data to file - hdf5io.createGroup("/data"); - hdf5io.setAttribute( - AQNWBIO::BaseDataType::I32, &data, "/data", "array", dataSize); + // single attribute + SECTION("single_value") + { + const signed int data = 1; + hdf5io.setAttribute( + AQNWBIO::BaseDataType::I32, &data, "/data", "single_value"); + } - hdf5io.close(); - // std::remove(filename.c_str()); -} + // integer array + SECTION("int_array") + { + const int data[] = {1, 2, 3, 4, 5}; + const int dataSize = sizeof(data) / sizeof(data[0]); -TEST_CASE("write_str_array", "[io]") -{ - std::string filename = getTestFilePath("test_str_array.h5"); - - HDF5IO hdf5io; - hdf5io.open(filename); + hdf5io.setAttribute( + AQNWBIO::BaseDataType::I32, &data, "/data", "array", dataSize); + } - // Setup data structures - const std::vector data = {"col1", "col2", "col3"}; + // integer array + SECTION("str_array") + { + const std::vector data = {"col1", "col2", "col3"}; - // Write data to file - hdf5io.createGroup("/data"); - hdf5io.setAttribute(data, "/data", "string_array"); + hdf5io.setAttribute(data, "/data", "string_array"); + } + // close file hdf5io.close(); - // std::remove(filename.c_str()); } -TEST_CASE("nwbfile_generation", "[nwb]") +TEST_CASE("generate_nwbfile", "[nwb]") { -std::string filename = getTestFilePath("test_nwb_file.h5"); + std::string filename = getTestFilePath("test_nwb_file.h5"); -NWBFile nwbfile(filename, "123", std::make_unique()); -nwbfile.open(); -nwbfile.close(); + NWBFile nwbfile("123", std::make_unique(filename)); + nwbfile.initialize(); + nwbfile.finalize(); }