From d30712e038d12949f52c3e94608a963b6a62ef92 Mon Sep 17 00:00:00 2001 From: Steph Prince <40640337+stephprince@users.noreply.github.com> Date: Mon, 22 Apr 2024 13:37:35 -0700 Subject: [PATCH 01/16] add validation gh action job --- .github/workflows/tests.yml | 39 +++++++++++++++++++++++++++++++++++++ tests/aq-nwb_test.cpp | 4 ++-- 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index fb2c89bc..fb2a2d87 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -57,3 +57,42 @@ jobs: - name: Test working-directory: build run: ctest --output-on-failure --no-tests=error -C Release -j 2 + + - name: Upload artifacts + uses: actions/upload-artifact@v3 + with: + name: test-files-${{ matrix.os }} + path: | + build/tests/data/*.nwb # TODO - check this is the correct path + + validate: + needs: tests + defaults: + run: + shell: bash + concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.os }} + cancel-in-progress: true + strategy: + fail-fast: false + matrix: + os: [macos-latest, ubuntu-latest] + + runs-on: ${{ matrix.os }} + + steps: + - name: Download test files + uses: actions/download-artifact@v3 + with: + name: test-files-${{ matrix.os }} + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install pynwb and run validation + run: | + python -m pip install --upgrade pip + python -m pip install pynwb + python -m pynwb.validate *.nwb \ No newline at end of file diff --git a/tests/aq-nwb_test.cpp b/tests/aq-nwb_test.cpp index fb2268a5..7149fee7 100644 --- a/tests/aq-nwb_test.cpp +++ b/tests/aq-nwb_test.cpp @@ -76,7 +76,7 @@ TEST_CASE("writeAttributes", "[hdf5io]") TEST_CASE("saveNWBFile", "[nwb]") { - std::string filename = getTestFilePath("test_nwb_file.h5"); + std::string filename = getTestFilePath("test_nwb_file.nwb"); NWB::NWBFile nwbfile("123", std::make_unique(filename)); nwbfile.initialize(); @@ -85,7 +85,7 @@ TEST_CASE("saveNWBFile", "[nwb]") TEST_CASE("startRecording", "[nwb]") { - std::string filename = getTestFilePath("test_recording.h5"); + std::string filename = getTestFilePath("test_recording.nwb"); NWB::NWBFile nwbfile("123", std::make_unique(filename)); nwbfile.initialize(); From d14bb136407adbc3403e7831c7b1076e8ce45fa5 Mon Sep 17 00:00:00 2001 From: Steph Prince <40640337+stephprince@users.noreply.github.com> Date: Mon, 22 Apr 2024 20:04:15 -0700 Subject: [PATCH 02/16] fix time formatting --- src/Utils.hpp | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/Utils.hpp b/src/Utils.hpp index 039596c6..65eb6ff5 100644 --- a/src/Utils.hpp +++ b/src/Utils.hpp @@ -36,7 +36,17 @@ inline std::string getCurrentTime() // Format the date and time in ISO 8601 format with the UTC offset std::ostringstream oss; - oss << std::put_time(&utcTime, "%FT%T%z"); + oss << std::put_time(&utcTime, "%FT%T"); + + // Get the timezone offset + auto localTime = std::localtime(¤tTime); + auto utcOffset = localTime->tm_hour - utcTime.tm_hour; + auto utcOffsetMinutes = (utcOffset < 0 ? -1 : 1) * (localTime->tm_min - utcTime.tm_min); + + // Add the timezone offset to the date and time string + oss << (utcOffset < 0 ? "-" : "+") + << std::setw(2) << std::setfill('0') << std::abs(utcOffset) << ":" + << std::setw(2) << std::setfill('0') << std::abs(utcOffsetMinutes); return oss.str(); } From fd4d8803567793404308ffbfe429584cb6014d7e Mon Sep 17 00:00:00 2001 From: Steph Prince <40640337+stephprince@users.noreply.github.com> Date: Mon, 22 Apr 2024 22:43:19 -0700 Subject: [PATCH 03/16] make filecreatedate a dataset --- src/BaseIO.hpp | 9 +++++++++ src/Utils.hpp | 9 +++++---- src/hdf5/HDF5IO.cpp | 24 ++++++++++++++++++++++++ src/hdf5/HDF5IO.hpp | 9 +++++++++ src/nwb/NWBFile.cpp | 2 +- 5 files changed, 48 insertions(+), 5 deletions(-) diff --git a/src/BaseIO.hpp b/src/BaseIO.hpp index cf867d18..51c9ecdc 100644 --- a/src/BaseIO.hpp +++ b/src/BaseIO.hpp @@ -217,6 +217,15 @@ class BaseIO virtual Status createStringDataSet(const std::string& path, const std::string& value) = 0; + /** + * @brief Creates a dataset that holds an array of a single datetime value. + * @param path The location in the file of the dataset. + * @param text The text value of the dataset. + * @return The status of the dataset creation operation. + */ + virtual Status createStringArrayDataSet(const std::string& path, + const std::string& text) = 0; + /** * @brief Creates a dataset that holds an array of references to groups within * the file. diff --git a/src/Utils.hpp b/src/Utils.hpp index 65eb6ff5..e32531e0 100644 --- a/src/Utils.hpp +++ b/src/Utils.hpp @@ -41,12 +41,13 @@ inline std::string getCurrentTime() // Get the timezone offset auto localTime = std::localtime(¤tTime); auto utcOffset = localTime->tm_hour - utcTime.tm_hour; - auto utcOffsetMinutes = (utcOffset < 0 ? -1 : 1) * (localTime->tm_min - utcTime.tm_min); + auto utcOffsetMinutes = + (utcOffset < 0 ? -1 : 1) * (localTime->tm_min - utcTime.tm_min); // Add the timezone offset to the date and time string - oss << (utcOffset < 0 ? "-" : "+") - << std::setw(2) << std::setfill('0') << std::abs(utcOffset) << ":" - << std::setw(2) << std::setfill('0') << std::abs(utcOffsetMinutes); + oss << (utcOffset < 0 ? "-" : "+") << std::setw(2) << std::setfill('0') + << std::abs(utcOffset) << ":" << std::setw(2) << std::setfill('0') + << std::abs(utcOffsetMinutes); return oss.str(); } diff --git a/src/hdf5/HDF5IO.cpp b/src/hdf5/HDF5IO.cpp index 3d098a32..8f4b565a 100644 --- a/src/hdf5/HDF5IO.cpp +++ b/src/hdf5/HDF5IO.cpp @@ -342,6 +342,30 @@ Status HDF5IO::createStringDataSet(const std::string& path, return Status::Success; } +Status HDF5IO::createStringArrayDataSet(const std::string& path, + const std::string& text) +{ + // std::unique_ptr dataset; + // DataType H5type = getH5Type(BaseDataType::STR(text.length())); + // // DataSpace dSpace(H5S_SCALAR); + + // DataSpace dSpace(1, 1, 1); + + // dataset = + // std::make_unique(file->createDataSet(path + "/" + name, + // H5type, dSpace)); + // dataset->writeDataBlock(1, BaseDataType::STR(text.length()), text.c_str()); + + std::unique_ptr dataset; + BaseDataType textType = BaseDataType::STR(text.length()); + + dataset = std::unique_ptr( + createDataSet(textType, SizeArray {0}, SizeArray {1}, path)); + dataset->writeDataBlock(1, textType, text.c_str()); + + return Status::Success; +} + AQNWB::BaseRecordingData* HDF5IO::getDataSet(const std::string& path) { std::unique_ptr data; diff --git a/src/hdf5/HDF5IO.hpp b/src/hdf5/HDF5IO.hpp index 06295260..b8a6cd94 100644 --- a/src/hdf5/HDF5IO.hpp +++ b/src/hdf5/HDF5IO.hpp @@ -153,6 +153,15 @@ class HDF5IO : public BaseIO Status createStringDataSet(const std::string& path, const std::string& value) override; + /** + * @brief Creates a dataset that holds an array of a single datetime value. + * @param path The location in the file of the dataset. + * @param text The text value of the dataset. + * @return The status of the dataset creation operation. + */ + Status createStringArrayDataSet(const std::string& path, + const std::string& text) override; + /** * @brief Creates a dataset that holds an array of references to groups within * the file. diff --git a/src/nwb/NWBFile.cpp b/src/nwb/NWBFile.cpp index 3da373bd..0e9e1588 100644 --- a/src/nwb/NWBFile.cpp +++ b/src/nwb/NWBFile.cpp @@ -63,7 +63,7 @@ Status NWBFile::createFileStructure() cacheSpecifications("hdmf-common/", HDMFVersion); std::string time = getCurrentTime(); - io->createStringDataSet("/file_create_date", time); // TODO - change to array + io->createStringArrayDataSet("/file_create_date", time); io->createStringDataSet("/session_description", "a recording session"); io->createStringDataSet("/session_start_time", time); io->createStringDataSet("/timestamps_reference_time", time); From 457fb28e1755f99ef2d689a7eba238ba2734a696 Mon Sep 17 00:00:00 2001 From: Steph Prince <40640337+stephprince@users.noreply.github.com> Date: Tue, 23 Apr 2024 11:48:07 -0700 Subject: [PATCH 04/16] update spec string generation for correct validation --- resources/generate_spec_files.py | 45 +++++++++++++++---- resources/spec/core/2.7.0/nwb.file.json | 2 +- resources/spec/core/2.7.0/nwb.namespace.json | 2 +- .../spec/hdmf-common/1.8.0/namespace.json | 2 +- .../spec/hdmf-common/1.8.0/resources.json | 1 - .../0.5.0}/experimental.json | 0 .../hdmf-experimental/0.5.0/namespace.json | 1 + .../hdmf-experimental/0.5.0/resources.json | 1 + 8 files changed, 41 insertions(+), 13 deletions(-) delete mode 100644 resources/spec/hdmf-common/1.8.0/resources.json rename resources/spec/{hdmf-common/1.8.0 => hdmf-experimental/0.5.0}/experimental.json (100%) create mode 100644 resources/spec/hdmf-experimental/0.5.0/namespace.json create mode 100644 resources/spec/hdmf-experimental/0.5.0/resources.json diff --git a/resources/generate_spec_files.py b/resources/generate_spec_files.py index 9150f754..4540b915 100644 --- a/resources/generate_spec_files.py +++ b/resources/generate_spec_files.py @@ -4,17 +4,44 @@ from ruamel.yaml import YAML # TODO - setup git submodule or cloning -schema_dir = Path('./resources/spec/core/2.7.0') +schema_dir = Path('./resources/schema/') -# create spec strings -for file in schema_dir.rglob(r"*.yaml"): +for file in schema_dir.rglob(r"*namespace.yaml"): # load file yaml = YAML(typ='safe') with open(file) as f: - spec = yaml.load(file) + namespace = yaml.load(file) - # convert to json - outfile = file.with_suffix('.json') - print(f'Generating file {outfile}') - with open(outfile, 'w') as fo: - json.dump(spec, fo, separators=(',', ':'),) \ No newline at end of file + # get all the sources + for i, ns in enumerate(namespace['namespaces']): + spec_dir = Path(f"./resources/spec/{ns['name']}/{ns['version']}") + spec_dir.mkdir(parents=True, exist_ok=True) + + # load and convert schema files + for s in ns['schema']: + if 'source' in s: + # load file + schema_file = file.parent / s['source'] + with open(schema_file) as f: + spec = yaml.load(schema_file) + + # convert to json + spec_file = (spec_dir / s['source']).with_suffix('.json') + print(f'Generating file {spec_file}') + with open(spec_file, 'w') as fo: + json.dump(spec, fo, separators=(',', ':'),) + + # reformat schema sources for namespace file + schema = list() + for s in ns['schema']: + if 'source' in s: + s = {'source': s['source'].split('.yaml')[0]} + schema.append(s) + ns['schema'] = schema + + # convert namespace json + ns_file = (spec_dir / file.name).with_suffix('.json') + ns_output = {'namespaces': [ns]} + print(f'Generating file {ns_file}') + with open(ns_file, 'w') as fo: + json.dump(ns_output, fo, separators=(',', ':'),) diff --git a/resources/spec/core/2.7.0/nwb.file.json b/resources/spec/core/2.7.0/nwb.file.json index 17f410cd..e0299048 100644 --- a/resources/spec/core/2.7.0/nwb.file.json +++ b/resources/spec/core/2.7.0/nwb.file.json @@ -1 +1 @@ -{"groups":[{"neurodata_type_def":"NWBFile","neurodata_type_inc":"NWBContainer","name":"root","doc":"An NWB file storing cellular-based neurophysiology data from a single experimental session.","attributes":[{"name":"nwb_version","dtype":"text","value":"2.7.0","doc":"File version string. Use semantic versioning, e.g. 1.2.1. This will be the name of the format with trailing major, minor and patch numbers."}],"datasets":[{"name":"file_create_date","dtype":"isodatetime","dims":["num_modifications"],"shape":[null],"doc":"A record of the date the file was created and of subsequent modifications. The date is stored in UTC with local timezone offset as ISO 8601 extended formatted strings: 2018-09-28T14:43:54.123+02:00. Dates stored in UTC end in \"Z\" with no timezone offset. Date accuracy is up to milliseconds. The file can be created after the experiment was run, so this may differ from the experiment start time. Each modification to the nwb file adds a new entry to the array."},{"name":"identifier","dtype":"text","doc":"A unique text identifier for the file. For example, concatenated lab name, file creation date/time and experimentalist, or a hash of these and/or other values. The goal is that the string should be unique to all other files."},{"name":"session_description","dtype":"text","doc":"A description of the experimental session and data in the file."},{"name":"session_start_time","dtype":"isodatetime","doc":"Date and time of the experiment/session start. The date is stored in UTC with local timezone offset as ISO 8601 extended formatted string: 2018-09-28T14:43:54.123+02:00. Dates stored in UTC end in \"Z\" with no timezone offset. Date accuracy is up to milliseconds."},{"name":"timestamps_reference_time","dtype":"isodatetime","doc":"Date and time corresponding to time zero of all timestamps. The date is stored in UTC with local timezone offset as ISO 8601 extended formatted string: 2018-09-28T14:43:54.123+02:00. Dates stored in UTC end in \"Z\" with no timezone offset. Date accuracy is up to milliseconds. All times stored in the file use this time as reference (i.e., time zero)."}],"groups":[{"name":"acquisition","doc":"Data streams recorded from the system, including ephys, ophys, tracking, etc. This group should be read-only after the experiment is completed and timestamps are corrected to a common timebase. The data stored here may be links to raw data stored in external NWB files. This will allow keeping bulky raw data out of the file while preserving the option of keeping some/all in the file. Acquired data includes tracking and experimental data streams (i.e., everything measured from the system). If bulky data is stored in the /acquisition group, the data can exist in a separate NWB file that is linked to by the file being used for processing and analysis.","groups":[{"neurodata_type_inc":"NWBDataInterface","doc":"Acquired, raw data.","quantity":"*"},{"neurodata_type_inc":"DynamicTable","doc":"Tabular data that is relevant to acquisition","quantity":"*"}]},{"name":"analysis","doc":"Lab-specific and custom scientific analysis of data. There is no defined format for the content of this group - the format is up to the individual user/lab. To facilitate sharing analysis data between labs, the contents here should be stored in standard types (e.g., neurodata_types) and appropriately documented. The file can store lab-specific and custom data analysis without restriction on its form or schema, reducing data formatting restrictions on end users. Such data should be placed in the analysis group. The analysis data should be documented so that it could be shared with other labs.","groups":[{"neurodata_type_inc":"NWBContainer","doc":"Custom analysis results.","quantity":"*"},{"neurodata_type_inc":"DynamicTable","doc":"Tabular data that is relevant to data stored in analysis","quantity":"*"}]},{"name":"scratch","doc":"A place to store one-off analysis results. Data placed here is not intended for sharing. By placing data here, users acknowledge that there is no guarantee that their data meets any standard.","quantity":"?","groups":[{"neurodata_type_inc":"NWBContainer","doc":"Any one-off containers","quantity":"*"},{"neurodata_type_inc":"DynamicTable","doc":"Any one-off tables","quantity":"*"}],"datasets":[{"neurodata_type_inc":"ScratchData","doc":"Any one-off datasets","quantity":"*"}]},{"name":"processing","doc":"The home for ProcessingModules. These modules perform intermediate analysis of data that is necessary to perform before scientific analysis. Examples include spike clustering, extracting position from tracking data, stitching together image slices. ProcessingModules can be large and express many data sets from relatively complex analysis (e.g., spike detection and clustering) or small, representing extraction of position information from tracking video, or even binary lick/no-lick decisions. Common software tools (e.g., klustakwik, MClust) are expected to read/write data here. 'Processing' refers to intermediate analysis of the acquired data to make it more amenable to scientific analysis.","groups":[{"neurodata_type_inc":"ProcessingModule","doc":"Intermediate analysis of acquired data.","quantity":"*"}]},{"name":"stimulus","doc":"Data pushed into the system (eg, video stimulus, sound, voltage, etc) and secondary representations of that data (eg, measurements of something used as a stimulus). This group should be made read-only after experiment complete and timestamps are corrected to common timebase. Stores both presented stimuli and stimulus templates, the latter in case the same stimulus is presented multiple times, or is pulled from an external stimulus library. Stimuli are here defined as any signal that is pushed into the system as part of the experiment (eg, sound, video, voltage, etc). Many different experiments can use the same stimuli, and stimuli can be re-used during an experiment. The stimulus group is organized so that one version of template stimuli can be stored and these be used multiple times. These templates can exist in the present file or can be linked to a remote library file.","groups":[{"name":"presentation","doc":"Stimuli presented during the experiment.","groups":[{"neurodata_type_inc":"TimeSeries","doc":"TimeSeries objects containing data of presented stimuli.","quantity":"*"},{"neurodata_type_inc":"NWBDataInterface","doc":"Generic NWB data interfaces, usually from an extension, containing data of presented stimuli.","quantity":"*"},{"neurodata_type_inc":"DynamicTable","doc":"DynamicTable objects containing data of presented stimuli.","quantity":"*"}]},{"name":"templates","doc":"Template stimuli. Timestamps in templates are based on stimulus design and are relative to the beginning of the stimulus. When templates are used, the stimulus instances must convert presentation times to the experiment`s time reference frame.","groups":[{"neurodata_type_inc":"TimeSeries","doc":"TimeSeries objects containing template data of presented stimuli.","quantity":"*"},{"neurodata_type_inc":"Images","doc":"Images objects containing images of presented stimuli.","quantity":"*"}]}]},{"name":"general","doc":"Experimental metadata, including protocol, notes and description of hardware device(s). The metadata stored in this section should be used to describe the experiment. Metadata necessary for interpreting the data is stored with the data. General experimental metadata, including animal strain, experimental protocols, experimenter, devices, etc, are stored under 'general'. Core metadata (e.g., that required to interpret data fields) is stored with the data itself, and implicitly defined by the file specification (e.g., time is in seconds). The strategy used here for storing non-core metadata is to use free-form text fields, such as would appear in sentences or paragraphs from a Methods section. Metadata fields are text to enable them to be more general, for example to represent ranges instead of numerical values. Machine-readable metadata is stored as attributes to these free-form datasets. All entries in the below table are to be included when data is present. Unused groups (e.g., intracellular_ephys in an optophysiology experiment) should not be created unless there is data to store within them.","datasets":[{"name":"data_collection","dtype":"text","doc":"Notes about data collection and analysis.","quantity":"?"},{"name":"experiment_description","dtype":"text","doc":"General description of the experiment.","quantity":"?"},{"name":"experimenter","dtype":"text","doc":"Name of person(s) who performed the experiment. Can also specify roles of different people involved.","quantity":"?","dims":["num_experimenters"],"shape":[null]},{"name":"institution","dtype":"text","doc":"Institution(s) where experiment was performed.","quantity":"?"},{"name":"keywords","dtype":"text","dims":["num_keywords"],"shape":[null],"doc":"Terms to search over.","quantity":"?"},{"name":"lab","dtype":"text","doc":"Laboratory where experiment was performed.","quantity":"?"},{"name":"notes","dtype":"text","doc":"Notes about the experiment.","quantity":"?"},{"name":"pharmacology","dtype":"text","doc":"Description of drugs used, including how and when they were administered. Anesthesia(s), painkiller(s), etc., plus dosage, concentration, etc.","quantity":"?"},{"name":"protocol","dtype":"text","doc":"Experimental protocol, if applicable. e.g., include IACUC protocol number.","quantity":"?"},{"name":"related_publications","dtype":"text","doc":"Publication information. PMID, DOI, URL, etc.","dims":["num_publications"],"shape":[null],"quantity":"?"},{"name":"session_id","dtype":"text","doc":"Lab-specific ID for the session.","quantity":"?"},{"name":"slices","dtype":"text","doc":"Description of slices, including information about preparation thickness, orientation, temperature, and bath solution.","quantity":"?"},{"name":"source_script","dtype":"text","doc":"Script file or link to public source code used to create this NWB file.","quantity":"?","attributes":[{"name":"file_name","dtype":"text","doc":"Name of script file."}]},{"name":"stimulus","dtype":"text","doc":"Notes about stimuli, such as how and where they were presented.","quantity":"?"},{"name":"surgery","dtype":"text","doc":"Narrative description about surgery/surgeries, including date(s) and who performed surgery.","quantity":"?"},{"name":"virus","dtype":"text","doc":"Information about virus(es) used in experiments, including virus ID, source, date made, injection location, volume, etc.","quantity":"?"}],"groups":[{"neurodata_type_inc":"LabMetaData","doc":"Place-holder than can be extended so that lab-specific meta-data can be placed in /general.","quantity":"*"},{"name":"devices","doc":"Description of hardware devices used during experiment, e.g., monitors, ADC boards, microscopes, etc.","quantity":"?","groups":[{"neurodata_type_inc":"Device","doc":"Data acquisition devices.","quantity":"*"}]},{"name":"subject","neurodata_type_inc":"Subject","doc":"Information about the animal or person from which the data was measured.","quantity":"?"},{"name":"extracellular_ephys","doc":"Metadata related to extracellular electrophysiology.","quantity":"?","groups":[{"neurodata_type_inc":"ElectrodeGroup","doc":"Physical group of electrodes.","quantity":"*"},{"name":"electrodes","neurodata_type_inc":"DynamicTable","doc":"A table of all electrodes (i.e. channels) used for recording.","quantity":"?","datasets":[{"name":"x","neurodata_type_inc":"VectorData","dtype":"float32","doc":"x coordinate of the channel location in the brain (+x is posterior).","quantity":"?"},{"name":"y","neurodata_type_inc":"VectorData","dtype":"float32","doc":"y coordinate of the channel location in the brain (+y is inferior).","quantity":"?"},{"name":"z","neurodata_type_inc":"VectorData","dtype":"float32","doc":"z coordinate of the channel location in the brain (+z is right).","quantity":"?"},{"name":"imp","neurodata_type_inc":"VectorData","dtype":"float32","doc":"Impedance of the channel, in ohms.","quantity":"?"},{"name":"location","neurodata_type_inc":"VectorData","dtype":"text","doc":"Location of the electrode (channel). Specify the area, layer, comments on estimation of area/layer, stereotaxic coordinates if in vivo, etc. Use standard atlas names for anatomical regions when possible."},{"name":"filtering","neurodata_type_inc":"VectorData","dtype":"text","doc":"Description of hardware filtering, including the filter name and frequency cutoffs.","quantity":"?"},{"name":"group","neurodata_type_inc":"VectorData","dtype":{"target_type":"ElectrodeGroup","reftype":"object"},"doc":"Reference to the ElectrodeGroup this electrode is a part of."},{"name":"group_name","neurodata_type_inc":"VectorData","dtype":"text","doc":"Name of the ElectrodeGroup this electrode is a part of."},{"name":"rel_x","neurodata_type_inc":"VectorData","dtype":"float32","doc":"x coordinate in electrode group","quantity":"?"},{"name":"rel_y","neurodata_type_inc":"VectorData","dtype":"float32","doc":"y coordinate in electrode group","quantity":"?"},{"name":"rel_z","neurodata_type_inc":"VectorData","dtype":"float32","doc":"z coordinate in electrode group","quantity":"?"},{"name":"reference","neurodata_type_inc":"VectorData","dtype":"text","doc":"Description of the reference electrode and/or reference scheme used for this electrode, e.g., \"stainless steel skull screw\" or \"online common average referencing\".","quantity":"?"}]}]},{"name":"intracellular_ephys","doc":"Metadata related to intracellular electrophysiology.","quantity":"?","datasets":[{"name":"filtering","dtype":"text","doc":"[DEPRECATED] Use IntracellularElectrode.filtering instead. Description of filtering used. Includes filtering type and parameters, frequency fall-off, etc. If this changes between TimeSeries, filter description should be stored as a text attribute for each TimeSeries.","quantity":"?"}],"groups":[{"neurodata_type_inc":"IntracellularElectrode","doc":"An intracellular electrode.","quantity":"*"},{"name":"sweep_table","neurodata_type_inc":"SweepTable","doc":"[DEPRECATED] Table used to group different PatchClampSeries. SweepTable is being replaced by IntracellularRecordingsTable and SimultaneousRecordingsTable tables. Additional SequentialRecordingsTable, RepetitionsTable and ExperimentalConditions tables provide enhanced support for experiment metadata.","quantity":"?"},{"name":"intracellular_recordings","neurodata_type_inc":"IntracellularRecordingsTable","doc":"A table to group together a stimulus and response from a single electrode and a single simultaneous recording. Each row in the table represents a single recording consisting typically of a stimulus and a corresponding response. In some cases, however, only a stimulus or a response are recorded as as part of an experiment. In this case both, the stimulus and response will point to the same TimeSeries while the idx_start and count of the invalid column will be set to -1, thus, indicating that no values have been recorded for the stimulus or response, respectively. Note, a recording MUST contain at least a stimulus or a response. Typically the stimulus and response are PatchClampSeries. However, the use of AD/DA channels that are not associated to an electrode is also common in intracellular electrophysiology, in which case other TimeSeries may be used.","quantity":"?"},{"name":"simultaneous_recordings","neurodata_type_inc":"SimultaneousRecordingsTable","doc":"A table for grouping different intracellular recordings from the IntracellularRecordingsTable table together that were recorded simultaneously from different electrodes","quantity":"?"},{"name":"sequential_recordings","neurodata_type_inc":"SequentialRecordingsTable","doc":"A table for grouping different sequential recordings from the SimultaneousRecordingsTable table together. This is typically used to group together sequential recordings where the a sequence of stimuli of the same type with varying parameters have been presented in a sequence.","quantity":"?"},{"name":"repetitions","neurodata_type_inc":"RepetitionsTable","doc":"A table for grouping different sequential intracellular recordings together. With each SequentialRecording typically representing a particular type of stimulus, the RepetitionsTable table is typically used to group sets of stimuli applied in sequence.","quantity":"?"},{"name":"experimental_conditions","neurodata_type_inc":"ExperimentalConditionsTable","doc":"A table for grouping different intracellular recording repetitions together that belong to the same experimental experimental_conditions.","quantity":"?"}]},{"name":"optogenetics","doc":"Metadata describing optogenetic stimuluation.","quantity":"?","groups":[{"neurodata_type_inc":"OptogeneticStimulusSite","doc":"An optogenetic stimulation site.","quantity":"*"}]},{"name":"optophysiology","doc":"Metadata related to optophysiology.","quantity":"?","groups":[{"neurodata_type_inc":"ImagingPlane","doc":"An imaging plane.","quantity":"*"}]}]},{"name":"intervals","doc":"Experimental intervals, whether that be logically distinct sub-experiments having a particular scientific goal, trials (see trials subgroup) during an experiment, or epochs (see epochs subgroup) deriving from analysis of data.","quantity":"?","groups":[{"name":"epochs","neurodata_type_inc":"TimeIntervals","doc":"Divisions in time marking experimental stages or sub-divisions of a single recording session.","quantity":"?"},{"name":"trials","neurodata_type_inc":"TimeIntervals","doc":"Repeated experimental events that have a logical grouping.","quantity":"?"},{"name":"invalid_times","neurodata_type_inc":"TimeIntervals","doc":"Time intervals that should be removed from analysis.","quantity":"?"},{"neurodata_type_inc":"TimeIntervals","doc":"Optional additional table(s) for describing other experimental time intervals.","quantity":"*"}]},{"name":"units","neurodata_type_inc":"Units","doc":"Data about sorted spike units.","quantity":"?"}]},{"neurodata_type_def":"LabMetaData","neurodata_type_inc":"NWBContainer","doc":"Lab-specific meta-data."},{"neurodata_type_def":"Subject","neurodata_type_inc":"NWBContainer","doc":"Information about the animal or person from which the data was measured.","datasets":[{"name":"age","dtype":"text","doc":"Age of subject. Can be supplied instead of 'date_of_birth'.","quantity":"?","attributes":[{"name":"reference","doc":"Age is with reference to this event. Can be 'birth' or 'gestational'. If reference is omitted, 'birth' is implied.","dtype":"text","required":false,"default_value":"birth"}]},{"name":"date_of_birth","dtype":"isodatetime","doc":"Date of birth of subject. Can be supplied instead of 'age'.","quantity":"?"},{"name":"description","dtype":"text","doc":"Description of subject and where subject came from (e.g., breeder, if animal).","quantity":"?"},{"name":"genotype","dtype":"text","doc":"Genetic strain. If absent, assume Wild Type (WT).","quantity":"?"},{"name":"sex","dtype":"text","doc":"Gender of subject.","quantity":"?"},{"name":"species","dtype":"text","doc":"Species of subject.","quantity":"?"},{"name":"strain","dtype":"text","doc":"Strain of subject.","quantity":"?"},{"name":"subject_id","dtype":"text","doc":"ID of animal/person used/participating in experiment (lab convention).","quantity":"?"},{"name":"weight","dtype":"text","doc":"Weight at time of experiment, at time of surgery and at other important times.","quantity":"?"}]}],"datasets":[{"neurodata_type_def":"ScratchData","neurodata_type_inc":"NWBData","doc":"Any one-off datasets","attributes":[{"name":"notes","doc":"Any notes the user has about the dataset being stored","dtype":"text"}]}]} \ No newline at end of file +{"groups":[{"neurodata_type_def":"NWBFile","neurodata_type_inc":"NWBContainer","name":"root","doc":"An NWB file storing cellular-based neurophysiology data from a single experimental session.","attributes":[{"name":"nwb_version","dtype":"text","value":"2.7.0-alpha","doc":"File version string. Use semantic versioning, e.g. 1.2.1. This will be the name of the format with trailing major, minor and patch numbers."}],"datasets":[{"name":"file_create_date","dtype":"isodatetime","dims":["num_modifications"],"shape":[null],"doc":"A record of the date the file was created and of subsequent modifications. The date is stored in UTC with local timezone offset as ISO 8601 extended formatted strings: 2018-09-28T14:43:54.123+02:00. Dates stored in UTC end in \"Z\" with no timezone offset. Date accuracy is up to milliseconds. The file can be created after the experiment was run, so this may differ from the experiment start time. Each modification to the nwb file adds a new entry to the array."},{"name":"identifier","dtype":"text","doc":"A unique text identifier for the file. For example, concatenated lab name, file creation date/time and experimentalist, or a hash of these and/or other values. The goal is that the string should be unique to all other files."},{"name":"session_description","dtype":"text","doc":"A description of the experimental session and data in the file."},{"name":"session_start_time","dtype":"isodatetime","doc":"Date and time of the experiment/session start. The date is stored in UTC with local timezone offset as ISO 8601 extended formatted string: 2018-09-28T14:43:54.123+02:00. Dates stored in UTC end in \"Z\" with no timezone offset. Date accuracy is up to milliseconds."},{"name":"timestamps_reference_time","dtype":"isodatetime","doc":"Date and time corresponding to time zero of all timestamps. The date is stored in UTC with local timezone offset as ISO 8601 extended formatted string: 2018-09-28T14:43:54.123+02:00. Dates stored in UTC end in \"Z\" with no timezone offset. Date accuracy is up to milliseconds. All times stored in the file use this time as reference (i.e., time zero)."}],"groups":[{"name":"acquisition","doc":"Data streams recorded from the system, including ephys, ophys, tracking, etc. This group should be read-only after the experiment is completed and timestamps are corrected to a common timebase. The data stored here may be links to raw data stored in external NWB files. This will allow keeping bulky raw data out of the file while preserving the option of keeping some/all in the file. Acquired data includes tracking and experimental data streams (i.e., everything measured from the system). If bulky data is stored in the /acquisition group, the data can exist in a separate NWB file that is linked to by the file being used for processing and analysis.","groups":[{"neurodata_type_inc":"NWBDataInterface","doc":"Acquired, raw data.","quantity":"*"},{"neurodata_type_inc":"DynamicTable","doc":"Tabular data that is relevant to acquisition","quantity":"*"}]},{"name":"analysis","doc":"Lab-specific and custom scientific analysis of data. There is no defined format for the content of this group - the format is up to the individual user/lab. To facilitate sharing analysis data between labs, the contents here should be stored in standard types (e.g., neurodata_types) and appropriately documented. The file can store lab-specific and custom data analysis without restriction on its form or schema, reducing data formatting restrictions on end users. Such data should be placed in the analysis group. The analysis data should be documented so that it could be shared with other labs.","groups":[{"neurodata_type_inc":"NWBContainer","doc":"Custom analysis results.","quantity":"*"},{"neurodata_type_inc":"DynamicTable","doc":"Tabular data that is relevant to data stored in analysis","quantity":"*"}]},{"name":"scratch","doc":"A place to store one-off analysis results. Data placed here is not intended for sharing. By placing data here, users acknowledge that there is no guarantee that their data meets any standard.","quantity":"?","groups":[{"neurodata_type_inc":"NWBContainer","doc":"Any one-off containers","quantity":"*"},{"neurodata_type_inc":"DynamicTable","doc":"Any one-off tables","quantity":"*"}],"datasets":[{"neurodata_type_inc":"ScratchData","doc":"Any one-off datasets","quantity":"*"}]},{"name":"processing","doc":"The home for ProcessingModules. These modules perform intermediate analysis of data that is necessary to perform before scientific analysis. Examples include spike clustering, extracting position from tracking data, stitching together image slices. ProcessingModules can be large and express many data sets from relatively complex analysis (e.g., spike detection and clustering) or small, representing extraction of position information from tracking video, or even binary lick/no-lick decisions. Common software tools (e.g., klustakwik, MClust) are expected to read/write data here. 'Processing' refers to intermediate analysis of the acquired data to make it more amenable to scientific analysis.","groups":[{"neurodata_type_inc":"ProcessingModule","doc":"Intermediate analysis of acquired data.","quantity":"*"}]},{"name":"stimulus","doc":"Data pushed into the system (eg, video stimulus, sound, voltage, etc) and secondary representations of that data (eg, measurements of something used as a stimulus). This group should be made read-only after experiment complete and timestamps are corrected to common timebase. Stores both presented stimuli and stimulus templates, the latter in case the same stimulus is presented multiple times, or is pulled from an external stimulus library. Stimuli are here defined as any signal that is pushed into the system as part of the experiment (eg, sound, video, voltage, etc). Many different experiments can use the same stimuli, and stimuli can be re-used during an experiment. The stimulus group is organized so that one version of template stimuli can be stored and these be used multiple times. These templates can exist in the present file or can be linked to a remote library file.","groups":[{"name":"presentation","doc":"Stimuli presented during the experiment.","groups":[{"neurodata_type_inc":"TimeSeries","doc":"TimeSeries objects containing data of presented stimuli.","quantity":"*"}]},{"name":"templates","doc":"Template stimuli. Timestamps in templates are based on stimulus design and are relative to the beginning of the stimulus. When templates are used, the stimulus instances must convert presentation times to the experiment`s time reference frame.","groups":[{"neurodata_type_inc":"TimeSeries","doc":"TimeSeries objects containing template data of presented stimuli.","quantity":"*"},{"neurodata_type_inc":"Images","doc":"Images objects containing images of presented stimuli.","quantity":"*"}]}]},{"name":"general","doc":"Experimental metadata, including protocol, notes and description of hardware device(s). The metadata stored in this section should be used to describe the experiment. Metadata necessary for interpreting the data is stored with the data. General experimental metadata, including animal strain, experimental protocols, experimenter, devices, etc, are stored under 'general'. Core metadata (e.g., that required to interpret data fields) is stored with the data itself, and implicitly defined by the file specification (e.g., time is in seconds). The strategy used here for storing non-core metadata is to use free-form text fields, such as would appear in sentences or paragraphs from a Methods section. Metadata fields are text to enable them to be more general, for example to represent ranges instead of numerical values. Machine-readable metadata is stored as attributes to these free-form datasets. All entries in the below table are to be included when data is present. Unused groups (e.g., intracellular_ephys in an optophysiology experiment) should not be created unless there is data to store within them.","datasets":[{"name":"data_collection","dtype":"text","doc":"Notes about data collection and analysis.","quantity":"?"},{"name":"experiment_description","dtype":"text","doc":"General description of the experiment.","quantity":"?"},{"name":"experimenter","dtype":"text","doc":"Name of person(s) who performed the experiment. Can also specify roles of different people involved.","quantity":"?","dims":["num_experimenters"],"shape":[null]},{"name":"institution","dtype":"text","doc":"Institution(s) where experiment was performed.","quantity":"?"},{"name":"keywords","dtype":"text","dims":["num_keywords"],"shape":[null],"doc":"Terms to search over.","quantity":"?"},{"name":"lab","dtype":"text","doc":"Laboratory where experiment was performed.","quantity":"?"},{"name":"notes","dtype":"text","doc":"Notes about the experiment.","quantity":"?"},{"name":"pharmacology","dtype":"text","doc":"Description of drugs used, including how and when they were administered. Anesthesia(s), painkiller(s), etc., plus dosage, concentration, etc.","quantity":"?"},{"name":"protocol","dtype":"text","doc":"Experimental protocol, if applicable. e.g., include IACUC protocol number.","quantity":"?"},{"name":"related_publications","dtype":"text","doc":"Publication information. PMID, DOI, URL, etc.","dims":["num_publications"],"shape":[null],"quantity":"?"},{"name":"session_id","dtype":"text","doc":"Lab-specific ID for the session.","quantity":"?"},{"name":"slices","dtype":"text","doc":"Description of slices, including information about preparation thickness, orientation, temperature, and bath solution.","quantity":"?"},{"name":"source_script","dtype":"text","doc":"Script file or link to public source code used to create this NWB file.","quantity":"?","attributes":[{"name":"file_name","dtype":"text","doc":"Name of script file."}]},{"name":"stimulus","dtype":"text","doc":"Notes about stimuli, such as how and where they were presented.","quantity":"?"},{"name":"surgery","dtype":"text","doc":"Narrative description about surgery/surgeries, including date(s) and who performed surgery.","quantity":"?"},{"name":"virus","dtype":"text","doc":"Information about virus(es) used in experiments, including virus ID, source, date made, injection location, volume, etc.","quantity":"?"}],"groups":[{"neurodata_type_inc":"LabMetaData","doc":"Place-holder than can be extended so that lab-specific meta-data can be placed in /general.","quantity":"*"},{"name":"devices","doc":"Description of hardware devices used during experiment, e.g., monitors, ADC boards, microscopes, etc.","quantity":"?","groups":[{"neurodata_type_inc":"Device","doc":"Data acquisition devices.","quantity":"*"}]},{"name":"subject","neurodata_type_inc":"Subject","doc":"Information about the animal or person from which the data was measured.","quantity":"?"},{"name":"extracellular_ephys","doc":"Metadata related to extracellular electrophysiology.","quantity":"?","groups":[{"neurodata_type_inc":"ElectrodeGroup","doc":"Physical group of electrodes.","quantity":"*"},{"name":"electrodes","neurodata_type_inc":"DynamicTable","doc":"A table of all electrodes (i.e. channels) used for recording.","quantity":"?","datasets":[{"name":"x","neurodata_type_inc":"VectorData","dtype":"float32","doc":"x coordinate of the channel location in the brain (+x is posterior).","quantity":"?"},{"name":"y","neurodata_type_inc":"VectorData","dtype":"float32","doc":"y coordinate of the channel location in the brain (+y is inferior).","quantity":"?"},{"name":"z","neurodata_type_inc":"VectorData","dtype":"float32","doc":"z coordinate of the channel location in the brain (+z is right).","quantity":"?"},{"name":"imp","neurodata_type_inc":"VectorData","dtype":"float32","doc":"Impedance of the channel, in ohms.","quantity":"?"},{"name":"location","neurodata_type_inc":"VectorData","dtype":"text","doc":"Location of the electrode (channel). Specify the area, layer, comments on estimation of area/layer, stereotaxic coordinates if in vivo, etc. Use standard atlas names for anatomical regions when possible."},{"name":"filtering","neurodata_type_inc":"VectorData","dtype":"text","doc":"Description of hardware filtering, including the filter name and frequency cutoffs.","quantity":"?"},{"name":"group","neurodata_type_inc":"VectorData","dtype":{"target_type":"ElectrodeGroup","reftype":"object"},"doc":"Reference to the ElectrodeGroup this electrode is a part of."},{"name":"group_name","neurodata_type_inc":"VectorData","dtype":"text","doc":"Name of the ElectrodeGroup this electrode is a part of."},{"name":"rel_x","neurodata_type_inc":"VectorData","dtype":"float32","doc":"x coordinate in electrode group","quantity":"?"},{"name":"rel_y","neurodata_type_inc":"VectorData","dtype":"float32","doc":"y coordinate in electrode group","quantity":"?"},{"name":"rel_z","neurodata_type_inc":"VectorData","dtype":"float32","doc":"z coordinate in electrode group","quantity":"?"},{"name":"reference","neurodata_type_inc":"VectorData","dtype":"text","doc":"Description of the reference electrode and/or reference scheme used for this electrode, e.g., \"stainless steel skull screw\" or \"online common average referencing\".","quantity":"?"}]}]},{"name":"intracellular_ephys","doc":"Metadata related to intracellular electrophysiology.","quantity":"?","datasets":[{"name":"filtering","dtype":"text","doc":"[DEPRECATED] Use IntracellularElectrode.filtering instead. Description of filtering used. Includes filtering type and parameters, frequency fall-off, etc. If this changes between TimeSeries, filter description should be stored as a text attribute for each TimeSeries.","quantity":"?"}],"groups":[{"neurodata_type_inc":"IntracellularElectrode","doc":"An intracellular electrode.","quantity":"*"},{"name":"sweep_table","neurodata_type_inc":"SweepTable","doc":"[DEPRECATED] Table used to group different PatchClampSeries. SweepTable is being replaced by IntracellularRecordingsTable and SimultaneousRecordingsTable tables. Additional SequentialRecordingsTable, RepetitionsTable and ExperimentalConditions tables provide enhanced support for experiment metadata.","quantity":"?"},{"name":"intracellular_recordings","neurodata_type_inc":"IntracellularRecordingsTable","doc":"A table to group together a stimulus and response from a single electrode and a single simultaneous recording. Each row in the table represents a single recording consisting typically of a stimulus and a corresponding response. In some cases, however, only a stimulus or a response are recorded as as part of an experiment. In this case both, the stimulus and response will point to the same TimeSeries while the idx_start and count of the invalid column will be set to -1, thus, indicating that no values have been recorded for the stimulus or response, respectively. Note, a recording MUST contain at least a stimulus or a response. Typically the stimulus and response are PatchClampSeries. However, the use of AD/DA channels that are not associated to an electrode is also common in intracellular electrophysiology, in which case other TimeSeries may be used.","quantity":"?"},{"name":"simultaneous_recordings","neurodata_type_inc":"SimultaneousRecordingsTable","doc":"A table for grouping different intracellular recordings from the IntracellularRecordingsTable table together that were recorded simultaneously from different electrodes","quantity":"?"},{"name":"sequential_recordings","neurodata_type_inc":"SequentialRecordingsTable","doc":"A table for grouping different sequential recordings from the SimultaneousRecordingsTable table together. This is typically used to group together sequential recordings where the a sequence of stimuli of the same type with varying parameters have been presented in a sequence.","quantity":"?"},{"name":"repetitions","neurodata_type_inc":"RepetitionsTable","doc":"A table for grouping different sequential intracellular recordings together. With each SequentialRecording typically representing a particular type of stimulus, the RepetitionsTable table is typically used to group sets of stimuli applied in sequence.","quantity":"?"},{"name":"experimental_conditions","neurodata_type_inc":"ExperimentalConditionsTable","doc":"A table for grouping different intracellular recording repetitions together that belong to the same experimental experimental_conditions.","quantity":"?"}]},{"name":"optogenetics","doc":"Metadata describing optogenetic stimuluation.","quantity":"?","groups":[{"neurodata_type_inc":"OptogeneticStimulusSite","doc":"An optogenetic stimulation site.","quantity":"*"}]},{"name":"optophysiology","doc":"Metadata related to optophysiology.","quantity":"?","groups":[{"neurodata_type_inc":"ImagingPlane","doc":"An imaging plane.","quantity":"*"}]}]},{"name":"intervals","doc":"Experimental intervals, whether that be logically distinct sub-experiments having a particular scientific goal, trials (see trials subgroup) during an experiment, or epochs (see epochs subgroup) deriving from analysis of data.","quantity":"?","groups":[{"name":"epochs","neurodata_type_inc":"TimeIntervals","doc":"Divisions in time marking experimental stages or sub-divisions of a single recording session.","quantity":"?"},{"name":"trials","neurodata_type_inc":"TimeIntervals","doc":"Repeated experimental events that have a logical grouping.","quantity":"?"},{"name":"invalid_times","neurodata_type_inc":"TimeIntervals","doc":"Time intervals that should be removed from analysis.","quantity":"?"},{"neurodata_type_inc":"TimeIntervals","doc":"Optional additional table(s) for describing other experimental time intervals.","quantity":"*"}]},{"name":"units","neurodata_type_inc":"Units","doc":"Data about sorted spike units.","quantity":"?"}]},{"neurodata_type_def":"LabMetaData","neurodata_type_inc":"NWBContainer","doc":"Lab-specific meta-data."},{"neurodata_type_def":"Subject","neurodata_type_inc":"NWBContainer","doc":"Information about the animal or person from which the data was measured.","datasets":[{"name":"age","dtype":"text","doc":"Age of subject. Can be supplied instead of 'date_of_birth'.","quantity":"?","attributes":[{"name":"reference","doc":"Age is with reference to this event. Can be 'birth' or 'gestational'. If reference is omitted, 'birth' is implied.","dtype":"text","required":false,"default_value":"birth"}]},{"name":"date_of_birth","dtype":"isodatetime","doc":"Date of birth of subject. Can be supplied instead of 'age'.","quantity":"?"},{"name":"description","dtype":"text","doc":"Description of subject and where subject came from (e.g., breeder, if animal).","quantity":"?"},{"name":"genotype","dtype":"text","doc":"Genetic strain. If absent, assume Wild Type (WT).","quantity":"?"},{"name":"sex","dtype":"text","doc":"Gender of subject.","quantity":"?"},{"name":"species","dtype":"text","doc":"Species of subject.","quantity":"?"},{"name":"strain","dtype":"text","doc":"Strain of subject.","quantity":"?"},{"name":"subject_id","dtype":"text","doc":"ID of animal/person used/participating in experiment (lab convention).","quantity":"?"},{"name":"weight","dtype":"text","doc":"Weight at time of experiment, at time of surgery and at other important times.","quantity":"?"}]}],"datasets":[{"neurodata_type_def":"ScratchData","neurodata_type_inc":"NWBData","doc":"Any one-off datasets","attributes":[{"name":"notes","doc":"Any notes the user has about the dataset being stored","dtype":"text"}]}]} \ No newline at end of file diff --git a/resources/spec/core/2.7.0/nwb.namespace.json b/resources/spec/core/2.7.0/nwb.namespace.json index 19cada18..1ebae4be 100644 --- a/resources/spec/core/2.7.0/nwb.namespace.json +++ b/resources/spec/core/2.7.0/nwb.namespace.json @@ -1 +1 @@ -{"namespaces":[{"name":"core","doc":"NWB namespace","author":["Andrew Tritt","Oliver Ruebel","Ryan Ly","Ben Dichter","Keith Godfrey","Jeff Teeters"],"contact":["ajtritt@lbl.gov","oruebel@lbl.gov","rly@lbl.gov","bdichter@lbl.gov","keithg@alleninstitute.org","jteeters@berkeley.edu"],"full_name":"NWB core","schema":[{"namespace":"hdmf-common"},{"doc":"This source module contains base data types used throughout the NWB data format.","source":"nwb.base.yaml","title":"Base data types"},{"doc":"This source module contains neurodata_types for device data.","source":"nwb.device.yaml","title":"Devices"},{"doc":"This source module contains neurodata_types for epoch data.","source":"nwb.epoch.yaml","title":"Epochs"},{"doc":"This source module contains neurodata_types for image data.","source":"nwb.image.yaml","title":"Image data"},{"doc":"Main NWB file specification.","source":"nwb.file.yaml","title":"NWB file"},{"doc":"Miscellaneous types.","source":"nwb.misc.yaml","title":"Miscellaneous neurodata_types."},{"doc":"This source module contains neurodata_types for behavior data.","source":"nwb.behavior.yaml","title":"Behavior"},{"doc":"This source module contains neurodata_types for extracellular electrophysiology data.","source":"nwb.ecephys.yaml","title":"Extracellular electrophysiology"},{"doc":"This source module contains neurodata_types for intracellular electrophysiology data.","source":"nwb.icephys.yaml","title":"Intracellular electrophysiology"},{"doc":"This source module contains neurodata_types for opto-genetics data.","source":"nwb.ogen.yaml","title":"Optogenetics"},{"doc":"This source module contains neurodata_types for optical physiology data.","source":"nwb.ophys.yaml","title":"Optical physiology"},{"doc":"This source module contains neurodata_type for retinotopy data.","source":"nwb.retinotopy.yaml","title":"Retinotopy"}],"version":"2.7.0"}]} \ No newline at end of file +{"namespaces":[{"name":"core","doc":"NWB namespace","author":["Andrew Tritt","Oliver Ruebel","Ryan Ly","Ben Dichter","Keith Godfrey","Jeff Teeters"],"contact":["ajtritt@lbl.gov","oruebel@lbl.gov","rly@lbl.gov","bdichter@lbl.gov","keithg@alleninstitute.org","jteeters@berkeley.edu"],"full_name":"NWB core","schema":[{"namespace":"hdmf-common"},{"source":"nwb.base"},{"source":"nwb.device"},{"source":"nwb.epoch"},{"source":"nwb.image"},{"source":"nwb.file"},{"source":"nwb.misc"},{"source":"nwb.behavior"},{"source":"nwb.ecephys"},{"source":"nwb.icephys"},{"source":"nwb.ogen"},{"source":"nwb.ophys"},{"source":"nwb.retinotopy"}],"version":"2.7.0"}]} \ No newline at end of file diff --git a/resources/spec/hdmf-common/1.8.0/namespace.json b/resources/spec/hdmf-common/1.8.0/namespace.json index 73a9a2c2..0b921a45 100644 --- a/resources/spec/hdmf-common/1.8.0/namespace.json +++ b/resources/spec/hdmf-common/1.8.0/namespace.json @@ -1 +1 @@ -{"namespaces":[{"name":"hdmf-common","doc":"Common data structures provided by HDMF","author":["Andrew Tritt","Oliver Ruebel","Ryan Ly","Ben Dichter"],"contact":["ajtritt@lbl.gov","oruebel@lbl.gov","rly@lbl.gov","bdichter@lbl.gov"],"full_name":"HDMF Common","schema":[{"doc":"base data types","source":"base.yaml","title":"Base data types"},{"doc":"data types for a column-based table","source":"table.yaml","title":"Table data types"},{"doc":"data types for different types of sparse matrices","source":"sparse.yaml","title":"Sparse data types"}],"version":"1.5.0"},{"name":"hdmf-experimental","doc":"Experimental data structures provided by HDMF. These are not guaranteed to be available in the future","author":["Andrew Tritt","Oliver Ruebel","Ryan Ly","Ben Dichter"],"contact":["ajtritt@lbl.gov","oruebel@lbl.gov","rly@lbl.gov","bdichter@lbl.gov"],"full_name":"HDMF Experimental","schema":[{"namespace":"hdmf-common"},{"doc":"Experimental data types","source":"experimental.yaml","title":"Experimental data types"},{"doc":"data types for storing references to web accessible resources","source":"resources.yaml","title":"Resource reference data types"}],"version":"0.1.0"}]} \ No newline at end of file +{"namespaces":[{"name":"hdmf-common","doc":"Common data structures provided by HDMF","author":["Andrew Tritt","Oliver Ruebel","Ryan Ly","Ben Dichter"],"contact":["ajtritt@lbl.gov","oruebel@lbl.gov","rly@lbl.gov","bdichter@lbl.gov"],"full_name":"HDMF Common","schema":[{"source":"base"},{"source":"table"},{"source":"sparse"}],"version":"1.8.0"}]} \ No newline at end of file diff --git a/resources/spec/hdmf-common/1.8.0/resources.json b/resources/spec/hdmf-common/1.8.0/resources.json deleted file mode 100644 index 846dcfa9..00000000 --- a/resources/spec/hdmf-common/1.8.0/resources.json +++ /dev/null @@ -1 +0,0 @@ -{"groups":[{"data_type_def":"ExternalResources","data_type_inc":"Container","doc":"A set of four tables for tracking external resource references in a file. NOTE: this data type is in beta testing and is subject to change in a later version.","datasets":[{"data_type_inc":"Data","name":"keys","doc":"A table for storing user terms that are used to refer to external resources.","dtype":[{"name":"key","dtype":"text","doc":"The user term that maps to one or more resources in the 'resources' table."}],"dims":["num_rows"],"shape":[null]},{"data_type_inc":"Data","name":"entities","doc":"A table for mapping user terms (i.e., keys) to resource entities.","dtype":[{"name":"keys_idx","dtype":"uint","doc":"The index to the key in the 'keys' table."},{"name":"resources_idx","dtype":"uint","doc":"The index into the 'resources' table"},{"name":"entity_id","dtype":"text","doc":"The unique identifier entity."},{"name":"entity_uri","dtype":"text","doc":"The URI for the entity this reference applies to. This can be an empty string."}],"dims":["num_rows"],"shape":[null]},{"data_type_inc":"Data","name":"resources","doc":"A table for mapping user terms (i.e., keys) to resource entities.","dtype":[{"name":"resource","dtype":"text","doc":"The name of the resource."},{"name":"resource_uri","dtype":"text","doc":"The URI for the resource. This can be an empty string."}],"dims":["num_rows"],"shape":[null]},{"data_type_inc":"Data","name":"objects","doc":"A table for identifying which objects in a file contain references to external resources.","dtype":[{"name":"object_id","dtype":"text","doc":"The UUID for the object."},{"name":"field","dtype":"text","doc":"The field of the object. This can be an empty string if the object is a dataset and the field is the dataset values."}],"dims":["num_rows"],"shape":[null]},{"data_type_inc":"Data","name":"object_keys","doc":"A table for identifying which objects use which keys.","dtype":[{"name":"objects_idx","dtype":"uint","doc":"The index to the 'objects' table for the object that holds the key."},{"name":"keys_idx","dtype":"uint","doc":"The index to the 'keys' table for the key."}],"dims":["num_rows"],"shape":[null]}]}]} \ No newline at end of file diff --git a/resources/spec/hdmf-common/1.8.0/experimental.json b/resources/spec/hdmf-experimental/0.5.0/experimental.json similarity index 100% rename from resources/spec/hdmf-common/1.8.0/experimental.json rename to resources/spec/hdmf-experimental/0.5.0/experimental.json diff --git a/resources/spec/hdmf-experimental/0.5.0/namespace.json b/resources/spec/hdmf-experimental/0.5.0/namespace.json new file mode 100644 index 00000000..26f1ac7d --- /dev/null +++ b/resources/spec/hdmf-experimental/0.5.0/namespace.json @@ -0,0 +1 @@ +{"namespaces":[{"name":"hdmf-experimental","doc":"Experimental data structures provided by HDMF. These are not guaranteed to be available in the future.","author":["Andrew Tritt","Oliver Ruebel","Ryan Ly","Ben Dichter","Matthew Avaylon"],"contact":["ajtritt@lbl.gov","oruebel@lbl.gov","rly@lbl.gov","bdichter@lbl.gov","mavaylon@lbl.gov"],"full_name":"HDMF Experimental","schema":[{"namespace":"hdmf-common"},{"source":"experimental"},{"source":"resources"}],"version":"0.5.0"}]} \ No newline at end of file diff --git a/resources/spec/hdmf-experimental/0.5.0/resources.json b/resources/spec/hdmf-experimental/0.5.0/resources.json new file mode 100644 index 00000000..0e7d71ff --- /dev/null +++ b/resources/spec/hdmf-experimental/0.5.0/resources.json @@ -0,0 +1 @@ +{"groups":[{"data_type_def":"HERD","data_type_inc":"Container","doc":"HDMF External Resources Data Structure. A set of six tables for tracking external resource references in a file or across multiple files.","datasets":[{"data_type_inc":"Data","name":"keys","doc":"A table for storing user terms that are used to refer to external resources.","dtype":[{"name":"key","dtype":"text","doc":"The user term that maps to one or more resources in the `resources` table, e.g., \"human\"."}],"dims":["num_rows"],"shape":[null]},{"data_type_inc":"Data","name":"files","doc":"A table for storing object ids of files used in external resources.","dtype":[{"name":"file_object_id","dtype":"text","doc":"The object id (UUID) of a file that contains objects that refers to external resources."}],"dims":["num_rows"],"shape":[null]},{"data_type_inc":"Data","name":"entities","doc":"A table for mapping user terms (i.e., keys) to resource entities.","dtype":[{"name":"entity_id","dtype":"text","doc":"The compact uniform resource identifier (CURIE) of the entity, in the form [prefix]:[unique local identifier], e.g., 'NCBI_TAXON:9606'."},{"name":"entity_uri","dtype":"text","doc":"The URI for the entity this reference applies to. This can be an empty string. e.g., https://www.ncbi.nlm.nih.gov/Taxonomy/Browser/wwwtax.cgi?mode=info&id=9606"}],"dims":["num_rows"],"shape":[null]},{"data_type_inc":"Data","name":"objects","doc":"A table for identifying which objects in a file contain references to external resources.","dtype":[{"name":"files_idx","dtype":"uint","doc":"The row index to the file in the `files` table containing the object."},{"name":"object_id","dtype":"text","doc":"The object id (UUID) of the object."},{"name":"object_type","dtype":"text","doc":"The data type of the object."},{"name":"relative_path","dtype":"text","doc":"The relative path from the data object with the `object_id` to the dataset or attribute with the value(s) that is associated with an external resource. This can be an empty string if the object is a dataset that contains the value(s) that is associated with an external resource."},{"name":"field","dtype":"text","doc":"The field within the compound data type using an external resource. This is used only if the dataset or attribute is a compound data type; otherwise this should be an empty string."}],"dims":["num_rows"],"shape":[null]},{"data_type_inc":"Data","name":"object_keys","doc":"A table for identifying which objects use which keys.","dtype":[{"name":"objects_idx","dtype":"uint","doc":"The row index to the object in the `objects` table that holds the key"},{"name":"keys_idx","dtype":"uint","doc":"The row index to the key in the `keys` table."}],"dims":["num_rows"],"shape":[null]},{"data_type_inc":"Data","name":"entity_keys","doc":"A table for identifying which keys use which entity.","dtype":[{"name":"entities_idx","dtype":"uint","doc":"The row index to the entity in the `entities` table."},{"name":"keys_idx","dtype":"uint","doc":"The row index to the key in the `keys` table."}],"dims":["num_rows"],"shape":[null]}]}]} \ No newline at end of file From bf0d7a5d871596cb584db4ff4e881f3cf564b40c Mon Sep 17 00:00:00 2001 From: Steph Prince <40640337+stephprince@users.noreply.github.com> Date: Tue, 23 Apr 2024 11:48:40 -0700 Subject: [PATCH 05/16] ignore schema yaml files --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 3b4c29a6..886586ae 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ # folders **/.DS_Store /tests/data/* +/resources/schema/* # IDEs .idea/ From c8d0045bb434297f465a2059ebe9f9346ddb36ec Mon Sep 17 00:00:00 2001 From: Steph Prince <40640337+stephprince@users.noreply.github.com> Date: Tue, 23 Apr 2024 11:49:23 -0700 Subject: [PATCH 06/16] add hdmf experimental spec group --- src/nwb/NWBFile.cpp | 2 ++ src/nwb/NWBFile.hpp | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/src/nwb/NWBFile.cpp b/src/nwb/NWBFile.cpp index 0e9e1588..e88a1043 100644 --- a/src/nwb/NWBFile.cpp +++ b/src/nwb/NWBFile.cpp @@ -59,8 +59,10 @@ Status NWBFile::createFileStructure() io->createGroup("general/extracellular_ephys"); io->createGroup("/specifications"); + io->createReferenceAttribute("/specifications", "/", ".specloc"); cacheSpecifications("core/", NWBVersion); cacheSpecifications("hdmf-common/", HDMFVersion); + cacheSpecifications("hdmf-experimental/", HDMFExperimentalVersion); std::string time = getCurrentTime(); io->createStringArrayDataSet("/file_create_date", time); diff --git a/src/nwb/NWBFile.hpp b/src/nwb/NWBFile.hpp index cba9e50a..4609b4ee 100644 --- a/src/nwb/NWBFile.hpp +++ b/src/nwb/NWBFile.hpp @@ -67,6 +67,11 @@ class NWBFile */ const std::string HDMFVersion = "1.8.0"; + /** + * @brief Indicates the HDMF experimental version. + */ + const std::string HDMFExperimentalVersion = "0.5.0"; + protected: /** * @brief Creates the default file structure. From 7938c300b0dbe8561d32f12cf0d7e56c801cb8e7 Mon Sep 17 00:00:00 2001 From: Steph Prince <40640337+stephprince@users.noreply.github.com> Date: Tue, 23 Apr 2024 11:51:41 -0700 Subject: [PATCH 07/16] remove comment in workflow --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index fb2a2d87..f97ab69e 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -63,7 +63,7 @@ jobs: with: name: test-files-${{ matrix.os }} path: | - build/tests/data/*.nwb # TODO - check this is the correct path + build/tests/data/*.nwb validate: needs: tests From e1bb55f1098c9e114407a6e967cd9b63e8bdb7e5 Mon Sep 17 00:00:00 2001 From: Steph Prince <40640337+stephprince@users.noreply.github.com> Date: Tue, 23 Apr 2024 11:56:05 -0700 Subject: [PATCH 08/16] fix formatting and comments --- .github/workflows/tests.yml | 2 +- src/BaseIO.hpp | 2 +- src/hdf5/HDF5IO.cpp | 11 ----------- src/hdf5/HDF5IO.hpp | 2 +- 4 files changed, 3 insertions(+), 14 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index f97ab69e..229bc3fc 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -95,4 +95,4 @@ jobs: run: | python -m pip install --upgrade pip python -m pip install pynwb - python -m pynwb.validate *.nwb \ No newline at end of file + python -m pynwb.validate *.nwb diff --git a/src/BaseIO.hpp b/src/BaseIO.hpp index 51c9ecdc..68cb02fa 100644 --- a/src/BaseIO.hpp +++ b/src/BaseIO.hpp @@ -218,7 +218,7 @@ class BaseIO const std::string& value) = 0; /** - * @brief Creates a dataset that holds an array of a single datetime value. + * @brief Creates a dataset that holds an array of a single string value. * @param path The location in the file of the dataset. * @param text The text value of the dataset. * @return The status of the dataset creation operation. diff --git a/src/hdf5/HDF5IO.cpp b/src/hdf5/HDF5IO.cpp index 8f4b565a..7484aa0a 100644 --- a/src/hdf5/HDF5IO.cpp +++ b/src/hdf5/HDF5IO.cpp @@ -345,17 +345,6 @@ Status HDF5IO::createStringDataSet(const std::string& path, Status HDF5IO::createStringArrayDataSet(const std::string& path, const std::string& text) { - // std::unique_ptr dataset; - // DataType H5type = getH5Type(BaseDataType::STR(text.length())); - // // DataSpace dSpace(H5S_SCALAR); - - // DataSpace dSpace(1, 1, 1); - - // dataset = - // std::make_unique(file->createDataSet(path + "/" + name, - // H5type, dSpace)); - // dataset->writeDataBlock(1, BaseDataType::STR(text.length()), text.c_str()); - std::unique_ptr dataset; BaseDataType textType = BaseDataType::STR(text.length()); diff --git a/src/hdf5/HDF5IO.hpp b/src/hdf5/HDF5IO.hpp index b8a6cd94..61b2186a 100644 --- a/src/hdf5/HDF5IO.hpp +++ b/src/hdf5/HDF5IO.hpp @@ -154,7 +154,7 @@ class HDF5IO : public BaseIO const std::string& value) override; /** - * @brief Creates a dataset that holds an array of a single datetime value. + * @brief Creates a dataset that holds an array of a single string value. * @param path The location in the file of the dataset. * @param text The text value of the dataset. * @return The status of the dataset creation operation. From 200388d7a2d088b158d2df7cdff7034fa8af4a17 Mon Sep 17 00:00:00 2001 From: Steph Prince <40640337+stephprince@users.noreply.github.com> Date: Wed, 24 Apr 2024 13:34:57 -0700 Subject: [PATCH 09/16] add vlen string flag --- src/BaseIO.cpp | 5 +++-- src/BaseIO.hpp | 15 +++++++++++++-- src/hdf5/HDF5IO.cpp | 44 ++++++++++++++++++++++++++++++++++++++------ src/hdf5/HDF5IO.hpp | 14 ++++++++++++-- src/nwb/NWBFile.cpp | 4 +++- 5 files changed, 69 insertions(+), 13 deletions(-) diff --git a/src/BaseIO.cpp b/src/BaseIO.cpp index f8c114b8..884b077f 100644 --- a/src/BaseIO.cpp +++ b/src/BaseIO.cpp @@ -71,7 +71,8 @@ BaseRecordingData::~BaseRecordingData() {} Status BaseRecordingData::writeDataBlock(const SizeType& xDataSize, const BaseDataType& type, - const void* data) + const void* data, + const bool isVlenStr) { - return writeDataBlock(xDataSize, size[1], type, data); + return writeDataBlock(xDataSize, size[1], type, data, isVlenStr); } diff --git a/src/BaseIO.hpp b/src/BaseIO.hpp index 68cb02fa..030faf7a 100644 --- a/src/BaseIO.hpp +++ b/src/BaseIO.hpp @@ -45,6 +45,7 @@ class BaseDataType T_F32, ///< 32-bit floating point T_F64, ///< 64-bit floating point T_STR, ///< String + V_STR, ///< Variable length string }; /** @@ -217,6 +218,14 @@ class BaseIO virtual Status createStringDataSet(const std::string& path, const std::string& value) = 0; + /** + * @brief Creates a dataset that holds an array of string values. + * @param path The location in the file of the dataset. + * @param values The vector of string values of the dataset. + * @return The status of the dataset creation operation. + */ + virtual Status createStringDataSet(const std::string& path, + const std::vector& values) = 0; /** * @brief Creates a dataset that holds an array of a single string value. * @param path The location in the file of the dataset. @@ -343,7 +352,8 @@ class BaseRecordingData */ Status writeDataBlock(const SizeType& xDataSize, const BaseDataType& type, - const void* data); + const void* data, + const bool isVlenStr = false); /** * @brief Writes a 2D block of data (samples x channels). @@ -356,7 +366,8 @@ class BaseRecordingData virtual Status writeDataBlock(const SizeType& xDataSize, const SizeType& yDataSize, const BaseDataType& type, - const void* data) = 0; + const void* data, + const bool isVlenStr = false) = 0; protected: /** diff --git a/src/hdf5/HDF5IO.cpp b/src/hdf5/HDF5IO.cpp index 7484aa0a..7746b309 100644 --- a/src/hdf5/HDF5IO.cpp +++ b/src/hdf5/HDF5IO.cpp @@ -342,15 +342,34 @@ Status HDF5IO::createStringDataSet(const std::string& path, return Status::Success; } +Status HDF5IO::createStringDataSet(const std::string& path, + const std::vector& values) +{ + if (!opened) + return Status::Failure; + + std::vector cStrs; + cStrs.reserve(values.size()); + for(const auto& str : values) { + cStrs.push_back(str.c_str()); + } + + std::unique_ptr dataset; + dataset = std::unique_ptr( + createDataSet(BaseDataType::V_STR, SizeArray {0}, SizeArray {1}, path)); + dataset->writeDataBlock(1, BaseDataType::V_STR, cStrs.data()); + + return Status::Success; +} + Status HDF5IO::createStringArrayDataSet(const std::string& path, - const std::string& text) + const std::string& text) { std::unique_ptr dataset; - BaseDataType textType = BaseDataType::STR(text.length()); dataset = std::unique_ptr( - createDataSet(textType, SizeArray {0}, SizeArray {1}, path)); - dataset->writeDataBlock(1, textType, text.c_str()); + createDataSet(BaseDataType::V_STR, SizeArray {0}, SizeArray {1}, path)); + dataset->writeDataBlock(1, BaseDataType::V_STR, text.c_str(), true); return Status::Success; } @@ -454,6 +473,9 @@ H5::DataType HDF5IO::getNativeType(BaseDataType type) case BaseDataType::Type::T_STR: return StrType(PredType::C_S1, type.typeSize); break; + case BaseDataType::Type::V_STR: + return StrType(PredType::C_S1, H5T_VARIABLE); + break; default: baseType = PredType::NATIVE_INT32; } @@ -502,6 +524,9 @@ H5::DataType HDF5IO::getH5Type(BaseDataType type) case BaseDataType::Type::T_STR: return StrType(PredType::C_S1, type.typeSize); break; + case BaseDataType::Type::V_STR: + return StrType(PredType::C_S1, H5T_VARIABLE); + break; default: return PredType::STD_I32LE; } @@ -554,7 +579,8 @@ HDF5RecordingData::~HDF5RecordingData() Status HDF5RecordingData::writeDataBlock(const SizeType& xDataSize, const SizeType& yDataSize, const BaseDataType& type, - const void* data) + const void* data, + const bool isVlenStr) { hsize_t dim[3], offset[3]; DataSpace fSpace; @@ -592,7 +618,13 @@ Status HDF5RecordingData::writeDataBlock(const SizeType& xDataSize, nativeType = HDF5IO::getNativeType(type); - dSet->write(data, nativeType, mSpace, fSpace); + if (isVlenStr) { + const char* cstr = static_cast(data); + std::string dataStr = cstr; + dSet->write(dataStr, nativeType, mSpace, fSpace); + } else { + dSet->write(data, nativeType, mSpace, fSpace); + } xPos += xDataSize; return Status::Success; diff --git a/src/hdf5/HDF5IO.hpp b/src/hdf5/HDF5IO.hpp index 61b2186a..37373a06 100644 --- a/src/hdf5/HDF5IO.hpp +++ b/src/hdf5/HDF5IO.hpp @@ -154,7 +154,16 @@ class HDF5IO : public BaseIO const std::string& value) override; /** - * @brief Creates a dataset that holds an array of a single string value. + * @brief Creates a dataset that holds an array of string values. + * @param path The location in the file of the dataset. + * @param values The vector of string values of the dataset. + * @return The status of the dataset creation operation. + */ + Status createStringDataSet(const std::string& path, + const std::vector& values) override; + + /** + * @brief Creates a dataset that holds an array of a vlen string value. * @param path The location in the file of the dataset. * @param text The text value of the dataset. * @return The status of the dataset creation operation. @@ -264,7 +273,8 @@ class HDF5RecordingData : public BaseRecordingData Status writeDataBlock(const SizeType& xDataSize, const SizeType& yDataSize, const BaseDataType& type, - const void* data); + const void* data, + const bool isVlenStr = false); /** * @brief Reads a block of data from the HDF5 dataset. diff --git a/src/nwb/NWBFile.cpp b/src/nwb/NWBFile.cpp index e88a1043..e84fb279 100644 --- a/src/nwb/NWBFile.cpp +++ b/src/nwb/NWBFile.cpp @@ -65,7 +65,9 @@ Status NWBFile::createFileStructure() cacheSpecifications("hdmf-experimental/", HDMFExperimentalVersion); std::string time = getCurrentTime(); - io->createStringArrayDataSet("/file_create_date", time); + // io->createStringArrayDataSet("/file_create_date", time); + std::vector timeVec = {time}; + io->createStringDataSet("/file_create_date", timeVec); io->createStringDataSet("/session_description", "a recording session"); io->createStringDataSet("/session_start_time", time); io->createStringDataSet("/timestamps_reference_time", time); From 0d4f0ac048a86f5e774c707de84c0acafaded653 Mon Sep 17 00:00:00 2001 From: Steph Prince <40640337+stephprince@users.noreply.github.com> Date: Wed, 24 Apr 2024 13:59:39 -0700 Subject: [PATCH 10/16] overload stringDataset function for string arrays --- src/BaseIO.cpp | 5 ++--- src/BaseIO.hpp | 14 ++------------ src/hdf5/HDF5IO.cpp | 25 +++---------------------- src/hdf5/HDF5IO.hpp | 12 +----------- src/nwb/NWBFile.cpp | 1 - 5 files changed, 8 insertions(+), 49 deletions(-) diff --git a/src/BaseIO.cpp b/src/BaseIO.cpp index 884b077f..f8c114b8 100644 --- a/src/BaseIO.cpp +++ b/src/BaseIO.cpp @@ -71,8 +71,7 @@ BaseRecordingData::~BaseRecordingData() {} Status BaseRecordingData::writeDataBlock(const SizeType& xDataSize, const BaseDataType& type, - const void* data, - const bool isVlenStr) + const void* data) { - return writeDataBlock(xDataSize, size[1], type, data, isVlenStr); + return writeDataBlock(xDataSize, size[1], type, data); } diff --git a/src/BaseIO.hpp b/src/BaseIO.hpp index 030faf7a..59c8b7a9 100644 --- a/src/BaseIO.hpp +++ b/src/BaseIO.hpp @@ -226,14 +226,6 @@ class BaseIO */ virtual Status createStringDataSet(const std::string& path, const std::vector& values) = 0; - /** - * @brief Creates a dataset that holds an array of a single string value. - * @param path The location in the file of the dataset. - * @param text The text value of the dataset. - * @return The status of the dataset creation operation. - */ - virtual Status createStringArrayDataSet(const std::string& path, - const std::string& text) = 0; /** * @brief Creates a dataset that holds an array of references to groups within @@ -352,8 +344,7 @@ class BaseRecordingData */ Status writeDataBlock(const SizeType& xDataSize, const BaseDataType& type, - const void* data, - const bool isVlenStr = false); + const void* data); /** * @brief Writes a 2D block of data (samples x channels). @@ -366,8 +357,7 @@ class BaseRecordingData virtual Status writeDataBlock(const SizeType& xDataSize, const SizeType& yDataSize, const BaseDataType& type, - const void* data, - const bool isVlenStr = false) = 0; + const void* data) = 0; protected: /** diff --git a/src/hdf5/HDF5IO.cpp b/src/hdf5/HDF5IO.cpp index 7746b309..995195e3 100644 --- a/src/hdf5/HDF5IO.cpp +++ b/src/hdf5/HDF5IO.cpp @@ -356,24 +356,12 @@ Status HDF5IO::createStringDataSet(const std::string& path, std::unique_ptr dataset; dataset = std::unique_ptr( - createDataSet(BaseDataType::V_STR, SizeArray {0}, SizeArray {1}, path)); + createDataSet(BaseDataType::V_STR, SizeArray{values.size()}, SizeArray{1}, path)); dataset->writeDataBlock(1, BaseDataType::V_STR, cStrs.data()); return Status::Success; } -Status HDF5IO::createStringArrayDataSet(const std::string& path, - const std::string& text) -{ - std::unique_ptr dataset; - - dataset = std::unique_ptr( - createDataSet(BaseDataType::V_STR, SizeArray {0}, SizeArray {1}, path)); - dataset->writeDataBlock(1, BaseDataType::V_STR, text.c_str(), true); - - return Status::Success; -} - AQNWB::BaseRecordingData* HDF5IO::getDataSet(const std::string& path) { std::unique_ptr data; @@ -579,8 +567,7 @@ HDF5RecordingData::~HDF5RecordingData() Status HDF5RecordingData::writeDataBlock(const SizeType& xDataSize, const SizeType& yDataSize, const BaseDataType& type, - const void* data, - const bool isVlenStr) + const void* data) { hsize_t dim[3], offset[3]; DataSpace fSpace; @@ -618,13 +605,7 @@ Status HDF5RecordingData::writeDataBlock(const SizeType& xDataSize, nativeType = HDF5IO::getNativeType(type); - if (isVlenStr) { - const char* cstr = static_cast(data); - std::string dataStr = cstr; - dSet->write(dataStr, nativeType, mSpace, fSpace); - } else { - dSet->write(data, nativeType, mSpace, fSpace); - } + dSet->write(data, nativeType, mSpace, fSpace); xPos += xDataSize; return Status::Success; diff --git a/src/hdf5/HDF5IO.hpp b/src/hdf5/HDF5IO.hpp index 37373a06..e7bdc5d8 100644 --- a/src/hdf5/HDF5IO.hpp +++ b/src/hdf5/HDF5IO.hpp @@ -162,15 +162,6 @@ class HDF5IO : public BaseIO Status createStringDataSet(const std::string& path, const std::vector& values) override; - /** - * @brief Creates a dataset that holds an array of a vlen string value. - * @param path The location in the file of the dataset. - * @param text The text value of the dataset. - * @return The status of the dataset creation operation. - */ - Status createStringArrayDataSet(const std::string& path, - const std::string& text) override; - /** * @brief Creates a dataset that holds an array of references to groups within * the file. @@ -273,8 +264,7 @@ class HDF5RecordingData : public BaseRecordingData Status writeDataBlock(const SizeType& xDataSize, const SizeType& yDataSize, const BaseDataType& type, - const void* data, - const bool isVlenStr = false); + const void* data); /** * @brief Reads a block of data from the HDF5 dataset. diff --git a/src/nwb/NWBFile.cpp b/src/nwb/NWBFile.cpp index e84fb279..268e6bbd 100644 --- a/src/nwb/NWBFile.cpp +++ b/src/nwb/NWBFile.cpp @@ -65,7 +65,6 @@ Status NWBFile::createFileStructure() cacheSpecifications("hdmf-experimental/", HDMFExperimentalVersion); std::string time = getCurrentTime(); - // io->createStringArrayDataSet("/file_create_date", time); std::vector timeVec = {time}; io->createStringDataSet("/file_create_date", timeVec); io->createStringDataSet("/session_description", "a recording session"); From 98b1affc92f38c9ab2c8a66d117eaf3ae284936e Mon Sep 17 00:00:00 2001 From: Steph Prince <40640337+stephprince@users.noreply.github.com> Date: Wed, 24 Apr 2024 14:39:53 -0700 Subject: [PATCH 11/16] fix formatting --- src/BaseIO.hpp | 4 ++-- src/hdf5/HDF5IO.cpp | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/BaseIO.hpp b/src/BaseIO.hpp index 59c8b7a9..15e1a718 100644 --- a/src/BaseIO.hpp +++ b/src/BaseIO.hpp @@ -224,8 +224,8 @@ class BaseIO * @param values The vector of string values of the dataset. * @return The status of the dataset creation operation. */ - virtual Status createStringDataSet(const std::string& path, - const std::vector& values) = 0; + virtual Status createStringDataSet( + const std::string& path, const std::vector& values) = 0; /** * @brief Creates a dataset that holds an array of references to groups within diff --git a/src/hdf5/HDF5IO.cpp b/src/hdf5/HDF5IO.cpp index 995195e3..f6ade19c 100644 --- a/src/hdf5/HDF5IO.cpp +++ b/src/hdf5/HDF5IO.cpp @@ -350,13 +350,13 @@ Status HDF5IO::createStringDataSet(const std::string& path, std::vector cStrs; cStrs.reserve(values.size()); - for(const auto& str : values) { - cStrs.push_back(str.c_str()); + for (const auto& str : values) { + cStrs.push_back(str.c_str()); } std::unique_ptr dataset; - dataset = std::unique_ptr( - createDataSet(BaseDataType::V_STR, SizeArray{values.size()}, SizeArray{1}, path)); + dataset = std::unique_ptr(createDataSet( + BaseDataType::V_STR, SizeArray {values.size()}, SizeArray {1}, path)); dataset->writeDataBlock(1, BaseDataType::V_STR, cStrs.data()); return Status::Success; From 129ce4bb154b8350ed32e1557dcf18dc7bcf11ba Mon Sep 17 00:00:00 2001 From: Steph Prince <40640337+stephprince@users.noreply.github.com> Date: Wed, 24 Apr 2024 15:24:49 -0700 Subject: [PATCH 12/16] update datetime to include tz offset --- src/Utils.hpp | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/Utils.hpp b/src/Utils.hpp index e32531e0..0362190f 100644 --- a/src/Utils.hpp +++ b/src/Utils.hpp @@ -30,20 +30,25 @@ inline std::string getCurrentTime() // Get current time auto currentTime = std::chrono::system_clock::to_time_t(std::chrono::system_clock::now()); - - // Convert to tm struct to extract date and time components std::tm utcTime = *std::gmtime(¤tTime); - // Format the date and time in ISO 8601 format with the UTC offset - std::ostringstream oss; - oss << std::put_time(&utcTime, "%FT%T"); - // Get the timezone offset auto localTime = std::localtime(¤tTime); auto utcOffset = localTime->tm_hour - utcTime.tm_hour; auto utcOffsetMinutes = (utcOffset < 0 ? -1 : 1) * (localTime->tm_min - utcTime.tm_min); + // Adjust the UTC time to the local time + utcTime.tm_hour += utcOffset; + if (utcTime.tm_hour < 0) { + utcTime.tm_hour += 24; + utcTime.tm_mday -= 1; + } + + // Format the date and time in ISO 8601 format with the UTC offset + std::ostringstream oss; + oss << std::put_time(&utcTime, "%FT%T"); + // Add the timezone offset to the date and time string oss << (utcOffset < 0 ? "-" : "+") << std::setw(2) << std::setfill('0') << std::abs(utcOffset) << ":" << std::setw(2) << std::setfill('0') From 5fea08448d84871fcb71372d57943b22e9b5e236 Mon Sep 17 00:00:00 2001 From: Steph Prince <40640337+stephprince@users.noreply.github.com> Date: Wed, 24 Apr 2024 15:43:48 -0700 Subject: [PATCH 13/16] update identifier and object_id for NWBFile to be unique --- src/nwb/NWBFile.cpp | 6 ++---- tests/aq-nwb_test.cpp | 5 +++-- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/nwb/NWBFile.cpp b/src/nwb/NWBFile.cpp index 268e6bbd..3a28f6eb 100644 --- a/src/nwb/NWBFile.cpp +++ b/src/nwb/NWBFile.cpp @@ -43,10 +43,8 @@ void NWBFile::finalize() Status NWBFile::createFileStructure() { - io->createAttribute("core", "/", "namespace"); - io->createAttribute("NWBFile", "/", "neurodata_type"); + io->createCommonNWBAttributes("/", "core", "NWBFile", ""); io->createAttribute(NWBVersion, "/", "nwb_version"); - io->createAttribute(identifierText, "/", "object_id"); io->createGroup("/acquisition"); io->createGroup("/analysis"); @@ -70,7 +68,7 @@ Status NWBFile::createFileStructure() io->createStringDataSet("/session_description", "a recording session"); io->createStringDataSet("/session_start_time", time); io->createStringDataSet("/timestamps_reference_time", time); - io->createStringDataSet("/identifier", "test-identifier"); + io->createStringDataSet("/identifier", identifierText); return Status::Success; } diff --git a/tests/aq-nwb_test.cpp b/tests/aq-nwb_test.cpp index 7149fee7..4a67ed13 100644 --- a/tests/aq-nwb_test.cpp +++ b/tests/aq-nwb_test.cpp @@ -7,6 +7,7 @@ #include "hdf5/HDF5IO.hpp" #include "nwb/NWBFile.hpp" #include "nwb/file/ElectrodeTable.hpp" +#include "Utils.hpp" using namespace AQNWB; namespace fs = std::filesystem; @@ -78,7 +79,7 @@ TEST_CASE("saveNWBFile", "[nwb]") { std::string filename = getTestFilePath("test_nwb_file.nwb"); - NWB::NWBFile nwbfile("123", std::make_unique(filename)); + NWB::NWBFile nwbfile(generateUuid(), std::make_unique(filename)); nwbfile.initialize(); nwbfile.finalize(); } @@ -87,7 +88,7 @@ TEST_CASE("startRecording", "[nwb]") { std::string filename = getTestFilePath("test_recording.nwb"); - NWB::NWBFile nwbfile("123", std::make_unique(filename)); + NWB::NWBFile nwbfile(generateUuid(), std::make_unique(filename)); nwbfile.initialize(); Status result = nwbfile.startRecording(); nwbfile.finalize(); From 1e78a6ee130edcacaf858bb579c0c2609e1cc41c Mon Sep 17 00:00:00 2001 From: Steph Prince <40640337+stephprince@users.noreply.github.com> Date: Wed, 24 Apr 2024 15:44:46 -0700 Subject: [PATCH 14/16] switch to nwbinspector for validation --- .github/workflows/tests.yml | 5 +++-- tests/aq-nwb_test.cpp | 8 +++++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 229bc3fc..14c55228 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -85,6 +85,7 @@ jobs: uses: actions/download-artifact@v3 with: name: test-files-${{ matrix.os }} + path: nwb_files - name: Set up Python uses: actions/setup-python@v5 @@ -94,5 +95,5 @@ jobs: - name: Install pynwb and run validation run: | python -m pip install --upgrade pip - python -m pip install pynwb - python -m pynwb.validate *.nwb + python -m pip install nwbinspector + nwbinspector nwb_files --threshold BEST_PRACTICE_VIOLATION diff --git a/tests/aq-nwb_test.cpp b/tests/aq-nwb_test.cpp index 4a67ed13..edc7bda1 100644 --- a/tests/aq-nwb_test.cpp +++ b/tests/aq-nwb_test.cpp @@ -4,10 +4,10 @@ #include #include "BaseIO.hpp" +#include "Utils.hpp" #include "hdf5/HDF5IO.hpp" #include "nwb/NWBFile.hpp" #include "nwb/file/ElectrodeTable.hpp" -#include "Utils.hpp" using namespace AQNWB; namespace fs = std::filesystem; @@ -79,7 +79,8 @@ TEST_CASE("saveNWBFile", "[nwb]") { std::string filename = getTestFilePath("test_nwb_file.nwb"); - NWB::NWBFile nwbfile(generateUuid(), std::make_unique(filename)); + NWB::NWBFile nwbfile(generateUuid(), + std::make_unique(filename)); nwbfile.initialize(); nwbfile.finalize(); } @@ -88,7 +89,8 @@ TEST_CASE("startRecording", "[nwb]") { std::string filename = getTestFilePath("test_recording.nwb"); - NWB::NWBFile nwbfile(generateUuid(), std::make_unique(filename)); + NWB::NWBFile nwbfile(generateUuid(), + std::make_unique(filename)); nwbfile.initialize(); Status result = nwbfile.startRecording(); nwbfile.finalize(); From 4b2434ae31c2ca137dfdfb2f16150e5b8f6a045c Mon Sep 17 00:00:00 2001 From: Steph Prince <40640337+stephprince@users.noreply.github.com> Date: Wed, 24 Apr 2024 17:26:05 -0700 Subject: [PATCH 15/16] update to use boost for currenttime --- src/Utils.hpp | 41 +++++++++++++++++------------------------ 1 file changed, 17 insertions(+), 24 deletions(-) diff --git a/src/Utils.hpp b/src/Utils.hpp index 0362190f..f2b0f453 100644 --- a/src/Utils.hpp +++ b/src/Utils.hpp @@ -6,6 +6,8 @@ #include #include #include +#include +#include "boost/date_time/c_local_time_adjustor.hpp" namespace AQNWB { @@ -27,33 +29,24 @@ inline std::string generateUuid() */ inline std::string getCurrentTime() { - // Get current time - auto currentTime = - std::chrono::system_clock::to_time_t(std::chrono::system_clock::now()); - std::tm utcTime = *std::gmtime(¤tTime); - - // Get the timezone offset - auto localTime = std::localtime(¤tTime); - auto utcOffset = localTime->tm_hour - utcTime.tm_hour; - auto utcOffsetMinutes = - (utcOffset < 0 ? -1 : 1) * (localTime->tm_min - utcTime.tm_min); - - // Adjust the UTC time to the local time - utcTime.tm_hour += utcOffset; - if (utcTime.tm_hour < 0) { - utcTime.tm_hour += 24; - utcTime.tm_mday -= 1; - } + // Set up boost time zone adjustment and time facet + using local_adj = boost::date_time::c_local_adjustor; + boost::posix_time::time_facet* f = new boost::posix_time::time_facet(); + f->time_duration_format("%+%H:%M"); + + // get local time, utc time, and offset + auto now = boost::posix_time::microsec_clock::universal_time(); + auto utc_now = local_adj::utc_to_local(now); + boost::posix_time::time_duration td = utc_now - now; // Format the date and time in ISO 8601 format with the UTC offset - std::ostringstream oss; - oss << std::put_time(&utcTime, "%FT%T"); + std::ostringstream oss_offset; + oss_offset.imbue(std::locale(oss_offset.getloc(), f)); + oss_offset << td; - // Add the timezone offset to the date and time string - oss << (utcOffset < 0 ? "-" : "+") << std::setw(2) << std::setfill('0') - << std::abs(utcOffset) << ":" << std::setw(2) << std::setfill('0') - << std::abs(utcOffsetMinutes); + std::string currentTime = to_iso_extended_string(utc_now); + currentTime += oss_offset.str(); - return oss.str(); + return currentTime; } } // namespace AQNWB From 601d640bf9fed951a8b8d6ed0873e0b7e00f640d Mon Sep 17 00:00:00 2001 From: Steph Prince <40640337+stephprince@users.noreply.github.com> Date: Wed, 24 Apr 2024 17:35:16 -0700 Subject: [PATCH 16/16] fix formatting --- src/Utils.hpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/Utils.hpp b/src/Utils.hpp index f2b0f453..a9c5fedf 100644 --- a/src/Utils.hpp +++ b/src/Utils.hpp @@ -3,10 +3,11 @@ #include #include +#include #include #include #include -#include + #include "boost/date_time/c_local_time_adjustor.hpp" namespace AQNWB @@ -30,7 +31,8 @@ inline std::string generateUuid() inline std::string getCurrentTime() { // Set up boost time zone adjustment and time facet - using local_adj = boost::date_time::c_local_adjustor; + using local_adj = + boost::date_time::c_local_adjustor; boost::posix_time::time_facet* f = new boost::posix_time::time_facet(); f->time_duration_format("%+%H:%M");