diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index fb2c89bc..14c55228 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -57,3 +57,43 @@ jobs: - name: Test working-directory: build run: ctest --output-on-failure --no-tests=error -C Release -j 2 + + - name: Upload artifacts + uses: actions/upload-artifact@v3 + with: + name: test-files-${{ matrix.os }} + path: | + build/tests/data/*.nwb + + validate: + needs: tests + defaults: + run: + shell: bash + concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.os }} + cancel-in-progress: true + strategy: + fail-fast: false + matrix: + os: [macos-latest, ubuntu-latest] + + runs-on: ${{ matrix.os }} + + steps: + - name: Download test files + uses: actions/download-artifact@v3 + with: + name: test-files-${{ matrix.os }} + path: nwb_files + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install pynwb and run validation + run: | + python -m pip install --upgrade pip + python -m pip install nwbinspector + nwbinspector nwb_files --threshold BEST_PRACTICE_VIOLATION diff --git a/.gitignore b/.gitignore index 4e65c38e..fa7554a6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ # folders **/.DS_Store /tests/data/* +/resources/schema/* # IDEs .idea/ diff --git a/resources/generate_spec_files.py b/resources/generate_spec_files.py index 9150f754..4540b915 100644 --- a/resources/generate_spec_files.py +++ b/resources/generate_spec_files.py @@ -4,17 +4,44 @@ from ruamel.yaml import YAML # TODO - setup git submodule or cloning -schema_dir = Path('./resources/spec/core/2.7.0') +schema_dir = Path('./resources/schema/') -# create spec strings -for file in schema_dir.rglob(r"*.yaml"): +for file in schema_dir.rglob(r"*namespace.yaml"): # load file yaml = YAML(typ='safe') with open(file) as f: - spec = yaml.load(file) + namespace = yaml.load(file) - # convert to json - outfile = file.with_suffix('.json') - print(f'Generating file {outfile}') - with open(outfile, 'w') as fo: - json.dump(spec, fo, separators=(',', ':'),) \ No newline at end of file + # get all the sources + for i, ns in enumerate(namespace['namespaces']): + spec_dir = Path(f"./resources/spec/{ns['name']}/{ns['version']}") + spec_dir.mkdir(parents=True, exist_ok=True) + + # load and convert schema files + for s in ns['schema']: + if 'source' in s: + # load file + schema_file = file.parent / s['source'] + with open(schema_file) as f: + spec = yaml.load(schema_file) + + # convert to json + spec_file = (spec_dir / s['source']).with_suffix('.json') + print(f'Generating file {spec_file}') + with open(spec_file, 'w') as fo: + json.dump(spec, fo, separators=(',', ':'),) + + # reformat schema sources for namespace file + schema = list() + for s in ns['schema']: + if 'source' in s: + s = {'source': s['source'].split('.yaml')[0]} + schema.append(s) + ns['schema'] = schema + + # convert namespace json + ns_file = (spec_dir / file.name).with_suffix('.json') + ns_output = {'namespaces': [ns]} + print(f'Generating file {ns_file}') + with open(ns_file, 'w') as fo: + json.dump(ns_output, fo, separators=(',', ':'),) diff --git a/resources/spec/core/2.7.0/nwb.file.json b/resources/spec/core/2.7.0/nwb.file.json index 17f410cd..e0299048 100644 --- a/resources/spec/core/2.7.0/nwb.file.json +++ b/resources/spec/core/2.7.0/nwb.file.json @@ -1 +1 @@ -{"groups":[{"neurodata_type_def":"NWBFile","neurodata_type_inc":"NWBContainer","name":"root","doc":"An NWB file storing cellular-based neurophysiology data from a single experimental session.","attributes":[{"name":"nwb_version","dtype":"text","value":"2.7.0","doc":"File version string. Use semantic versioning, e.g. 1.2.1. This will be the name of the format with trailing major, minor and patch numbers."}],"datasets":[{"name":"file_create_date","dtype":"isodatetime","dims":["num_modifications"],"shape":[null],"doc":"A record of the date the file was created and of subsequent modifications. The date is stored in UTC with local timezone offset as ISO 8601 extended formatted strings: 2018-09-28T14:43:54.123+02:00. Dates stored in UTC end in \"Z\" with no timezone offset. Date accuracy is up to milliseconds. The file can be created after the experiment was run, so this may differ from the experiment start time. Each modification to the nwb file adds a new entry to the array."},{"name":"identifier","dtype":"text","doc":"A unique text identifier for the file. For example, concatenated lab name, file creation date/time and experimentalist, or a hash of these and/or other values. The goal is that the string should be unique to all other files."},{"name":"session_description","dtype":"text","doc":"A description of the experimental session and data in the file."},{"name":"session_start_time","dtype":"isodatetime","doc":"Date and time of the experiment/session start. The date is stored in UTC with local timezone offset as ISO 8601 extended formatted string: 2018-09-28T14:43:54.123+02:00. Dates stored in UTC end in \"Z\" with no timezone offset. Date accuracy is up to milliseconds."},{"name":"timestamps_reference_time","dtype":"isodatetime","doc":"Date and time corresponding to time zero of all timestamps. The date is stored in UTC with local timezone offset as ISO 8601 extended formatted string: 2018-09-28T14:43:54.123+02:00. Dates stored in UTC end in \"Z\" with no timezone offset. Date accuracy is up to milliseconds. All times stored in the file use this time as reference (i.e., time zero)."}],"groups":[{"name":"acquisition","doc":"Data streams recorded from the system, including ephys, ophys, tracking, etc. This group should be read-only after the experiment is completed and timestamps are corrected to a common timebase. The data stored here may be links to raw data stored in external NWB files. This will allow keeping bulky raw data out of the file while preserving the option of keeping some/all in the file. Acquired data includes tracking and experimental data streams (i.e., everything measured from the system). If bulky data is stored in the /acquisition group, the data can exist in a separate NWB file that is linked to by the file being used for processing and analysis.","groups":[{"neurodata_type_inc":"NWBDataInterface","doc":"Acquired, raw data.","quantity":"*"},{"neurodata_type_inc":"DynamicTable","doc":"Tabular data that is relevant to acquisition","quantity":"*"}]},{"name":"analysis","doc":"Lab-specific and custom scientific analysis of data. There is no defined format for the content of this group - the format is up to the individual user/lab. To facilitate sharing analysis data between labs, the contents here should be stored in standard types (e.g., neurodata_types) and appropriately documented. The file can store lab-specific and custom data analysis without restriction on its form or schema, reducing data formatting restrictions on end users. Such data should be placed in the analysis group. The analysis data should be documented so that it could be shared with other labs.","groups":[{"neurodata_type_inc":"NWBContainer","doc":"Custom analysis results.","quantity":"*"},{"neurodata_type_inc":"DynamicTable","doc":"Tabular data that is relevant to data stored in analysis","quantity":"*"}]},{"name":"scratch","doc":"A place to store one-off analysis results. Data placed here is not intended for sharing. By placing data here, users acknowledge that there is no guarantee that their data meets any standard.","quantity":"?","groups":[{"neurodata_type_inc":"NWBContainer","doc":"Any one-off containers","quantity":"*"},{"neurodata_type_inc":"DynamicTable","doc":"Any one-off tables","quantity":"*"}],"datasets":[{"neurodata_type_inc":"ScratchData","doc":"Any one-off datasets","quantity":"*"}]},{"name":"processing","doc":"The home for ProcessingModules. These modules perform intermediate analysis of data that is necessary to perform before scientific analysis. Examples include spike clustering, extracting position from tracking data, stitching together image slices. ProcessingModules can be large and express many data sets from relatively complex analysis (e.g., spike detection and clustering) or small, representing extraction of position information from tracking video, or even binary lick/no-lick decisions. Common software tools (e.g., klustakwik, MClust) are expected to read/write data here. 'Processing' refers to intermediate analysis of the acquired data to make it more amenable to scientific analysis.","groups":[{"neurodata_type_inc":"ProcessingModule","doc":"Intermediate analysis of acquired data.","quantity":"*"}]},{"name":"stimulus","doc":"Data pushed into the system (eg, video stimulus, sound, voltage, etc) and secondary representations of that data (eg, measurements of something used as a stimulus). This group should be made read-only after experiment complete and timestamps are corrected to common timebase. Stores both presented stimuli and stimulus templates, the latter in case the same stimulus is presented multiple times, or is pulled from an external stimulus library. Stimuli are here defined as any signal that is pushed into the system as part of the experiment (eg, sound, video, voltage, etc). Many different experiments can use the same stimuli, and stimuli can be re-used during an experiment. The stimulus group is organized so that one version of template stimuli can be stored and these be used multiple times. These templates can exist in the present file or can be linked to a remote library file.","groups":[{"name":"presentation","doc":"Stimuli presented during the experiment.","groups":[{"neurodata_type_inc":"TimeSeries","doc":"TimeSeries objects containing data of presented stimuli.","quantity":"*"},{"neurodata_type_inc":"NWBDataInterface","doc":"Generic NWB data interfaces, usually from an extension, containing data of presented stimuli.","quantity":"*"},{"neurodata_type_inc":"DynamicTable","doc":"DynamicTable objects containing data of presented stimuli.","quantity":"*"}]},{"name":"templates","doc":"Template stimuli. Timestamps in templates are based on stimulus design and are relative to the beginning of the stimulus. When templates are used, the stimulus instances must convert presentation times to the experiment`s time reference frame.","groups":[{"neurodata_type_inc":"TimeSeries","doc":"TimeSeries objects containing template data of presented stimuli.","quantity":"*"},{"neurodata_type_inc":"Images","doc":"Images objects containing images of presented stimuli.","quantity":"*"}]}]},{"name":"general","doc":"Experimental metadata, including protocol, notes and description of hardware device(s). The metadata stored in this section should be used to describe the experiment. Metadata necessary for interpreting the data is stored with the data. General experimental metadata, including animal strain, experimental protocols, experimenter, devices, etc, are stored under 'general'. Core metadata (e.g., that required to interpret data fields) is stored with the data itself, and implicitly defined by the file specification (e.g., time is in seconds). The strategy used here for storing non-core metadata is to use free-form text fields, such as would appear in sentences or paragraphs from a Methods section. Metadata fields are text to enable them to be more general, for example to represent ranges instead of numerical values. Machine-readable metadata is stored as attributes to these free-form datasets. All entries in the below table are to be included when data is present. Unused groups (e.g., intracellular_ephys in an optophysiology experiment) should not be created unless there is data to store within them.","datasets":[{"name":"data_collection","dtype":"text","doc":"Notes about data collection and analysis.","quantity":"?"},{"name":"experiment_description","dtype":"text","doc":"General description of the experiment.","quantity":"?"},{"name":"experimenter","dtype":"text","doc":"Name of person(s) who performed the experiment. Can also specify roles of different people involved.","quantity":"?","dims":["num_experimenters"],"shape":[null]},{"name":"institution","dtype":"text","doc":"Institution(s) where experiment was performed.","quantity":"?"},{"name":"keywords","dtype":"text","dims":["num_keywords"],"shape":[null],"doc":"Terms to search over.","quantity":"?"},{"name":"lab","dtype":"text","doc":"Laboratory where experiment was performed.","quantity":"?"},{"name":"notes","dtype":"text","doc":"Notes about the experiment.","quantity":"?"},{"name":"pharmacology","dtype":"text","doc":"Description of drugs used, including how and when they were administered. Anesthesia(s), painkiller(s), etc., plus dosage, concentration, etc.","quantity":"?"},{"name":"protocol","dtype":"text","doc":"Experimental protocol, if applicable. e.g., include IACUC protocol number.","quantity":"?"},{"name":"related_publications","dtype":"text","doc":"Publication information. PMID, DOI, URL, etc.","dims":["num_publications"],"shape":[null],"quantity":"?"},{"name":"session_id","dtype":"text","doc":"Lab-specific ID for the session.","quantity":"?"},{"name":"slices","dtype":"text","doc":"Description of slices, including information about preparation thickness, orientation, temperature, and bath solution.","quantity":"?"},{"name":"source_script","dtype":"text","doc":"Script file or link to public source code used to create this NWB file.","quantity":"?","attributes":[{"name":"file_name","dtype":"text","doc":"Name of script file."}]},{"name":"stimulus","dtype":"text","doc":"Notes about stimuli, such as how and where they were presented.","quantity":"?"},{"name":"surgery","dtype":"text","doc":"Narrative description about surgery/surgeries, including date(s) and who performed surgery.","quantity":"?"},{"name":"virus","dtype":"text","doc":"Information about virus(es) used in experiments, including virus ID, source, date made, injection location, volume, etc.","quantity":"?"}],"groups":[{"neurodata_type_inc":"LabMetaData","doc":"Place-holder than can be extended so that lab-specific meta-data can be placed in /general.","quantity":"*"},{"name":"devices","doc":"Description of hardware devices used during experiment, e.g., monitors, ADC boards, microscopes, etc.","quantity":"?","groups":[{"neurodata_type_inc":"Device","doc":"Data acquisition devices.","quantity":"*"}]},{"name":"subject","neurodata_type_inc":"Subject","doc":"Information about the animal or person from which the data was measured.","quantity":"?"},{"name":"extracellular_ephys","doc":"Metadata related to extracellular electrophysiology.","quantity":"?","groups":[{"neurodata_type_inc":"ElectrodeGroup","doc":"Physical group of electrodes.","quantity":"*"},{"name":"electrodes","neurodata_type_inc":"DynamicTable","doc":"A table of all electrodes (i.e. channels) used for recording.","quantity":"?","datasets":[{"name":"x","neurodata_type_inc":"VectorData","dtype":"float32","doc":"x coordinate of the channel location in the brain (+x is posterior).","quantity":"?"},{"name":"y","neurodata_type_inc":"VectorData","dtype":"float32","doc":"y coordinate of the channel location in the brain (+y is inferior).","quantity":"?"},{"name":"z","neurodata_type_inc":"VectorData","dtype":"float32","doc":"z coordinate of the channel location in the brain (+z is right).","quantity":"?"},{"name":"imp","neurodata_type_inc":"VectorData","dtype":"float32","doc":"Impedance of the channel, in ohms.","quantity":"?"},{"name":"location","neurodata_type_inc":"VectorData","dtype":"text","doc":"Location of the electrode (channel). Specify the area, layer, comments on estimation of area/layer, stereotaxic coordinates if in vivo, etc. Use standard atlas names for anatomical regions when possible."},{"name":"filtering","neurodata_type_inc":"VectorData","dtype":"text","doc":"Description of hardware filtering, including the filter name and frequency cutoffs.","quantity":"?"},{"name":"group","neurodata_type_inc":"VectorData","dtype":{"target_type":"ElectrodeGroup","reftype":"object"},"doc":"Reference to the ElectrodeGroup this electrode is a part of."},{"name":"group_name","neurodata_type_inc":"VectorData","dtype":"text","doc":"Name of the ElectrodeGroup this electrode is a part of."},{"name":"rel_x","neurodata_type_inc":"VectorData","dtype":"float32","doc":"x coordinate in electrode group","quantity":"?"},{"name":"rel_y","neurodata_type_inc":"VectorData","dtype":"float32","doc":"y coordinate in electrode group","quantity":"?"},{"name":"rel_z","neurodata_type_inc":"VectorData","dtype":"float32","doc":"z coordinate in electrode group","quantity":"?"},{"name":"reference","neurodata_type_inc":"VectorData","dtype":"text","doc":"Description of the reference electrode and/or reference scheme used for this electrode, e.g., \"stainless steel skull screw\" or \"online common average referencing\".","quantity":"?"}]}]},{"name":"intracellular_ephys","doc":"Metadata related to intracellular electrophysiology.","quantity":"?","datasets":[{"name":"filtering","dtype":"text","doc":"[DEPRECATED] Use IntracellularElectrode.filtering instead. Description of filtering used. Includes filtering type and parameters, frequency fall-off, etc. If this changes between TimeSeries, filter description should be stored as a text attribute for each TimeSeries.","quantity":"?"}],"groups":[{"neurodata_type_inc":"IntracellularElectrode","doc":"An intracellular electrode.","quantity":"*"},{"name":"sweep_table","neurodata_type_inc":"SweepTable","doc":"[DEPRECATED] Table used to group different PatchClampSeries. SweepTable is being replaced by IntracellularRecordingsTable and SimultaneousRecordingsTable tables. Additional SequentialRecordingsTable, RepetitionsTable and ExperimentalConditions tables provide enhanced support for experiment metadata.","quantity":"?"},{"name":"intracellular_recordings","neurodata_type_inc":"IntracellularRecordingsTable","doc":"A table to group together a stimulus and response from a single electrode and a single simultaneous recording. Each row in the table represents a single recording consisting typically of a stimulus and a corresponding response. In some cases, however, only a stimulus or a response are recorded as as part of an experiment. In this case both, the stimulus and response will point to the same TimeSeries while the idx_start and count of the invalid column will be set to -1, thus, indicating that no values have been recorded for the stimulus or response, respectively. Note, a recording MUST contain at least a stimulus or a response. Typically the stimulus and response are PatchClampSeries. However, the use of AD/DA channels that are not associated to an electrode is also common in intracellular electrophysiology, in which case other TimeSeries may be used.","quantity":"?"},{"name":"simultaneous_recordings","neurodata_type_inc":"SimultaneousRecordingsTable","doc":"A table for grouping different intracellular recordings from the IntracellularRecordingsTable table together that were recorded simultaneously from different electrodes","quantity":"?"},{"name":"sequential_recordings","neurodata_type_inc":"SequentialRecordingsTable","doc":"A table for grouping different sequential recordings from the SimultaneousRecordingsTable table together. This is typically used to group together sequential recordings where the a sequence of stimuli of the same type with varying parameters have been presented in a sequence.","quantity":"?"},{"name":"repetitions","neurodata_type_inc":"RepetitionsTable","doc":"A table for grouping different sequential intracellular recordings together. With each SequentialRecording typically representing a particular type of stimulus, the RepetitionsTable table is typically used to group sets of stimuli applied in sequence.","quantity":"?"},{"name":"experimental_conditions","neurodata_type_inc":"ExperimentalConditionsTable","doc":"A table for grouping different intracellular recording repetitions together that belong to the same experimental experimental_conditions.","quantity":"?"}]},{"name":"optogenetics","doc":"Metadata describing optogenetic stimuluation.","quantity":"?","groups":[{"neurodata_type_inc":"OptogeneticStimulusSite","doc":"An optogenetic stimulation site.","quantity":"*"}]},{"name":"optophysiology","doc":"Metadata related to optophysiology.","quantity":"?","groups":[{"neurodata_type_inc":"ImagingPlane","doc":"An imaging plane.","quantity":"*"}]}]},{"name":"intervals","doc":"Experimental intervals, whether that be logically distinct sub-experiments having a particular scientific goal, trials (see trials subgroup) during an experiment, or epochs (see epochs subgroup) deriving from analysis of data.","quantity":"?","groups":[{"name":"epochs","neurodata_type_inc":"TimeIntervals","doc":"Divisions in time marking experimental stages or sub-divisions of a single recording session.","quantity":"?"},{"name":"trials","neurodata_type_inc":"TimeIntervals","doc":"Repeated experimental events that have a logical grouping.","quantity":"?"},{"name":"invalid_times","neurodata_type_inc":"TimeIntervals","doc":"Time intervals that should be removed from analysis.","quantity":"?"},{"neurodata_type_inc":"TimeIntervals","doc":"Optional additional table(s) for describing other experimental time intervals.","quantity":"*"}]},{"name":"units","neurodata_type_inc":"Units","doc":"Data about sorted spike units.","quantity":"?"}]},{"neurodata_type_def":"LabMetaData","neurodata_type_inc":"NWBContainer","doc":"Lab-specific meta-data."},{"neurodata_type_def":"Subject","neurodata_type_inc":"NWBContainer","doc":"Information about the animal or person from which the data was measured.","datasets":[{"name":"age","dtype":"text","doc":"Age of subject. Can be supplied instead of 'date_of_birth'.","quantity":"?","attributes":[{"name":"reference","doc":"Age is with reference to this event. Can be 'birth' or 'gestational'. If reference is omitted, 'birth' is implied.","dtype":"text","required":false,"default_value":"birth"}]},{"name":"date_of_birth","dtype":"isodatetime","doc":"Date of birth of subject. Can be supplied instead of 'age'.","quantity":"?"},{"name":"description","dtype":"text","doc":"Description of subject and where subject came from (e.g., breeder, if animal).","quantity":"?"},{"name":"genotype","dtype":"text","doc":"Genetic strain. If absent, assume Wild Type (WT).","quantity":"?"},{"name":"sex","dtype":"text","doc":"Gender of subject.","quantity":"?"},{"name":"species","dtype":"text","doc":"Species of subject.","quantity":"?"},{"name":"strain","dtype":"text","doc":"Strain of subject.","quantity":"?"},{"name":"subject_id","dtype":"text","doc":"ID of animal/person used/participating in experiment (lab convention).","quantity":"?"},{"name":"weight","dtype":"text","doc":"Weight at time of experiment, at time of surgery and at other important times.","quantity":"?"}]}],"datasets":[{"neurodata_type_def":"ScratchData","neurodata_type_inc":"NWBData","doc":"Any one-off datasets","attributes":[{"name":"notes","doc":"Any notes the user has about the dataset being stored","dtype":"text"}]}]} \ No newline at end of file +{"groups":[{"neurodata_type_def":"NWBFile","neurodata_type_inc":"NWBContainer","name":"root","doc":"An NWB file storing cellular-based neurophysiology data from a single experimental session.","attributes":[{"name":"nwb_version","dtype":"text","value":"2.7.0-alpha","doc":"File version string. Use semantic versioning, e.g. 1.2.1. This will be the name of the format with trailing major, minor and patch numbers."}],"datasets":[{"name":"file_create_date","dtype":"isodatetime","dims":["num_modifications"],"shape":[null],"doc":"A record of the date the file was created and of subsequent modifications. The date is stored in UTC with local timezone offset as ISO 8601 extended formatted strings: 2018-09-28T14:43:54.123+02:00. Dates stored in UTC end in \"Z\" with no timezone offset. Date accuracy is up to milliseconds. The file can be created after the experiment was run, so this may differ from the experiment start time. Each modification to the nwb file adds a new entry to the array."},{"name":"identifier","dtype":"text","doc":"A unique text identifier for the file. For example, concatenated lab name, file creation date/time and experimentalist, or a hash of these and/or other values. The goal is that the string should be unique to all other files."},{"name":"session_description","dtype":"text","doc":"A description of the experimental session and data in the file."},{"name":"session_start_time","dtype":"isodatetime","doc":"Date and time of the experiment/session start. The date is stored in UTC with local timezone offset as ISO 8601 extended formatted string: 2018-09-28T14:43:54.123+02:00. Dates stored in UTC end in \"Z\" with no timezone offset. Date accuracy is up to milliseconds."},{"name":"timestamps_reference_time","dtype":"isodatetime","doc":"Date and time corresponding to time zero of all timestamps. The date is stored in UTC with local timezone offset as ISO 8601 extended formatted string: 2018-09-28T14:43:54.123+02:00. Dates stored in UTC end in \"Z\" with no timezone offset. Date accuracy is up to milliseconds. All times stored in the file use this time as reference (i.e., time zero)."}],"groups":[{"name":"acquisition","doc":"Data streams recorded from the system, including ephys, ophys, tracking, etc. This group should be read-only after the experiment is completed and timestamps are corrected to a common timebase. The data stored here may be links to raw data stored in external NWB files. This will allow keeping bulky raw data out of the file while preserving the option of keeping some/all in the file. Acquired data includes tracking and experimental data streams (i.e., everything measured from the system). If bulky data is stored in the /acquisition group, the data can exist in a separate NWB file that is linked to by the file being used for processing and analysis.","groups":[{"neurodata_type_inc":"NWBDataInterface","doc":"Acquired, raw data.","quantity":"*"},{"neurodata_type_inc":"DynamicTable","doc":"Tabular data that is relevant to acquisition","quantity":"*"}]},{"name":"analysis","doc":"Lab-specific and custom scientific analysis of data. There is no defined format for the content of this group - the format is up to the individual user/lab. To facilitate sharing analysis data between labs, the contents here should be stored in standard types (e.g., neurodata_types) and appropriately documented. The file can store lab-specific and custom data analysis without restriction on its form or schema, reducing data formatting restrictions on end users. Such data should be placed in the analysis group. The analysis data should be documented so that it could be shared with other labs.","groups":[{"neurodata_type_inc":"NWBContainer","doc":"Custom analysis results.","quantity":"*"},{"neurodata_type_inc":"DynamicTable","doc":"Tabular data that is relevant to data stored in analysis","quantity":"*"}]},{"name":"scratch","doc":"A place to store one-off analysis results. Data placed here is not intended for sharing. By placing data here, users acknowledge that there is no guarantee that their data meets any standard.","quantity":"?","groups":[{"neurodata_type_inc":"NWBContainer","doc":"Any one-off containers","quantity":"*"},{"neurodata_type_inc":"DynamicTable","doc":"Any one-off tables","quantity":"*"}],"datasets":[{"neurodata_type_inc":"ScratchData","doc":"Any one-off datasets","quantity":"*"}]},{"name":"processing","doc":"The home for ProcessingModules. These modules perform intermediate analysis of data that is necessary to perform before scientific analysis. Examples include spike clustering, extracting position from tracking data, stitching together image slices. ProcessingModules can be large and express many data sets from relatively complex analysis (e.g., spike detection and clustering) or small, representing extraction of position information from tracking video, or even binary lick/no-lick decisions. Common software tools (e.g., klustakwik, MClust) are expected to read/write data here. 'Processing' refers to intermediate analysis of the acquired data to make it more amenable to scientific analysis.","groups":[{"neurodata_type_inc":"ProcessingModule","doc":"Intermediate analysis of acquired data.","quantity":"*"}]},{"name":"stimulus","doc":"Data pushed into the system (eg, video stimulus, sound, voltage, etc) and secondary representations of that data (eg, measurements of something used as a stimulus). This group should be made read-only after experiment complete and timestamps are corrected to common timebase. Stores both presented stimuli and stimulus templates, the latter in case the same stimulus is presented multiple times, or is pulled from an external stimulus library. Stimuli are here defined as any signal that is pushed into the system as part of the experiment (eg, sound, video, voltage, etc). Many different experiments can use the same stimuli, and stimuli can be re-used during an experiment. The stimulus group is organized so that one version of template stimuli can be stored and these be used multiple times. These templates can exist in the present file or can be linked to a remote library file.","groups":[{"name":"presentation","doc":"Stimuli presented during the experiment.","groups":[{"neurodata_type_inc":"TimeSeries","doc":"TimeSeries objects containing data of presented stimuli.","quantity":"*"}]},{"name":"templates","doc":"Template stimuli. Timestamps in templates are based on stimulus design and are relative to the beginning of the stimulus. When templates are used, the stimulus instances must convert presentation times to the experiment`s time reference frame.","groups":[{"neurodata_type_inc":"TimeSeries","doc":"TimeSeries objects containing template data of presented stimuli.","quantity":"*"},{"neurodata_type_inc":"Images","doc":"Images objects containing images of presented stimuli.","quantity":"*"}]}]},{"name":"general","doc":"Experimental metadata, including protocol, notes and description of hardware device(s). The metadata stored in this section should be used to describe the experiment. Metadata necessary for interpreting the data is stored with the data. General experimental metadata, including animal strain, experimental protocols, experimenter, devices, etc, are stored under 'general'. Core metadata (e.g., that required to interpret data fields) is stored with the data itself, and implicitly defined by the file specification (e.g., time is in seconds). The strategy used here for storing non-core metadata is to use free-form text fields, such as would appear in sentences or paragraphs from a Methods section. Metadata fields are text to enable them to be more general, for example to represent ranges instead of numerical values. Machine-readable metadata is stored as attributes to these free-form datasets. All entries in the below table are to be included when data is present. Unused groups (e.g., intracellular_ephys in an optophysiology experiment) should not be created unless there is data to store within them.","datasets":[{"name":"data_collection","dtype":"text","doc":"Notes about data collection and analysis.","quantity":"?"},{"name":"experiment_description","dtype":"text","doc":"General description of the experiment.","quantity":"?"},{"name":"experimenter","dtype":"text","doc":"Name of person(s) who performed the experiment. Can also specify roles of different people involved.","quantity":"?","dims":["num_experimenters"],"shape":[null]},{"name":"institution","dtype":"text","doc":"Institution(s) where experiment was performed.","quantity":"?"},{"name":"keywords","dtype":"text","dims":["num_keywords"],"shape":[null],"doc":"Terms to search over.","quantity":"?"},{"name":"lab","dtype":"text","doc":"Laboratory where experiment was performed.","quantity":"?"},{"name":"notes","dtype":"text","doc":"Notes about the experiment.","quantity":"?"},{"name":"pharmacology","dtype":"text","doc":"Description of drugs used, including how and when they were administered. Anesthesia(s), painkiller(s), etc., plus dosage, concentration, etc.","quantity":"?"},{"name":"protocol","dtype":"text","doc":"Experimental protocol, if applicable. e.g., include IACUC protocol number.","quantity":"?"},{"name":"related_publications","dtype":"text","doc":"Publication information. PMID, DOI, URL, etc.","dims":["num_publications"],"shape":[null],"quantity":"?"},{"name":"session_id","dtype":"text","doc":"Lab-specific ID for the session.","quantity":"?"},{"name":"slices","dtype":"text","doc":"Description of slices, including information about preparation thickness, orientation, temperature, and bath solution.","quantity":"?"},{"name":"source_script","dtype":"text","doc":"Script file or link to public source code used to create this NWB file.","quantity":"?","attributes":[{"name":"file_name","dtype":"text","doc":"Name of script file."}]},{"name":"stimulus","dtype":"text","doc":"Notes about stimuli, such as how and where they were presented.","quantity":"?"},{"name":"surgery","dtype":"text","doc":"Narrative description about surgery/surgeries, including date(s) and who performed surgery.","quantity":"?"},{"name":"virus","dtype":"text","doc":"Information about virus(es) used in experiments, including virus ID, source, date made, injection location, volume, etc.","quantity":"?"}],"groups":[{"neurodata_type_inc":"LabMetaData","doc":"Place-holder than can be extended so that lab-specific meta-data can be placed in /general.","quantity":"*"},{"name":"devices","doc":"Description of hardware devices used during experiment, e.g., monitors, ADC boards, microscopes, etc.","quantity":"?","groups":[{"neurodata_type_inc":"Device","doc":"Data acquisition devices.","quantity":"*"}]},{"name":"subject","neurodata_type_inc":"Subject","doc":"Information about the animal or person from which the data was measured.","quantity":"?"},{"name":"extracellular_ephys","doc":"Metadata related to extracellular electrophysiology.","quantity":"?","groups":[{"neurodata_type_inc":"ElectrodeGroup","doc":"Physical group of electrodes.","quantity":"*"},{"name":"electrodes","neurodata_type_inc":"DynamicTable","doc":"A table of all electrodes (i.e. channels) used for recording.","quantity":"?","datasets":[{"name":"x","neurodata_type_inc":"VectorData","dtype":"float32","doc":"x coordinate of the channel location in the brain (+x is posterior).","quantity":"?"},{"name":"y","neurodata_type_inc":"VectorData","dtype":"float32","doc":"y coordinate of the channel location in the brain (+y is inferior).","quantity":"?"},{"name":"z","neurodata_type_inc":"VectorData","dtype":"float32","doc":"z coordinate of the channel location in the brain (+z is right).","quantity":"?"},{"name":"imp","neurodata_type_inc":"VectorData","dtype":"float32","doc":"Impedance of the channel, in ohms.","quantity":"?"},{"name":"location","neurodata_type_inc":"VectorData","dtype":"text","doc":"Location of the electrode (channel). Specify the area, layer, comments on estimation of area/layer, stereotaxic coordinates if in vivo, etc. Use standard atlas names for anatomical regions when possible."},{"name":"filtering","neurodata_type_inc":"VectorData","dtype":"text","doc":"Description of hardware filtering, including the filter name and frequency cutoffs.","quantity":"?"},{"name":"group","neurodata_type_inc":"VectorData","dtype":{"target_type":"ElectrodeGroup","reftype":"object"},"doc":"Reference to the ElectrodeGroup this electrode is a part of."},{"name":"group_name","neurodata_type_inc":"VectorData","dtype":"text","doc":"Name of the ElectrodeGroup this electrode is a part of."},{"name":"rel_x","neurodata_type_inc":"VectorData","dtype":"float32","doc":"x coordinate in electrode group","quantity":"?"},{"name":"rel_y","neurodata_type_inc":"VectorData","dtype":"float32","doc":"y coordinate in electrode group","quantity":"?"},{"name":"rel_z","neurodata_type_inc":"VectorData","dtype":"float32","doc":"z coordinate in electrode group","quantity":"?"},{"name":"reference","neurodata_type_inc":"VectorData","dtype":"text","doc":"Description of the reference electrode and/or reference scheme used for this electrode, e.g., \"stainless steel skull screw\" or \"online common average referencing\".","quantity":"?"}]}]},{"name":"intracellular_ephys","doc":"Metadata related to intracellular electrophysiology.","quantity":"?","datasets":[{"name":"filtering","dtype":"text","doc":"[DEPRECATED] Use IntracellularElectrode.filtering instead. Description of filtering used. Includes filtering type and parameters, frequency fall-off, etc. If this changes between TimeSeries, filter description should be stored as a text attribute for each TimeSeries.","quantity":"?"}],"groups":[{"neurodata_type_inc":"IntracellularElectrode","doc":"An intracellular electrode.","quantity":"*"},{"name":"sweep_table","neurodata_type_inc":"SweepTable","doc":"[DEPRECATED] Table used to group different PatchClampSeries. SweepTable is being replaced by IntracellularRecordingsTable and SimultaneousRecordingsTable tables. Additional SequentialRecordingsTable, RepetitionsTable and ExperimentalConditions tables provide enhanced support for experiment metadata.","quantity":"?"},{"name":"intracellular_recordings","neurodata_type_inc":"IntracellularRecordingsTable","doc":"A table to group together a stimulus and response from a single electrode and a single simultaneous recording. Each row in the table represents a single recording consisting typically of a stimulus and a corresponding response. In some cases, however, only a stimulus or a response are recorded as as part of an experiment. In this case both, the stimulus and response will point to the same TimeSeries while the idx_start and count of the invalid column will be set to -1, thus, indicating that no values have been recorded for the stimulus or response, respectively. Note, a recording MUST contain at least a stimulus or a response. Typically the stimulus and response are PatchClampSeries. However, the use of AD/DA channels that are not associated to an electrode is also common in intracellular electrophysiology, in which case other TimeSeries may be used.","quantity":"?"},{"name":"simultaneous_recordings","neurodata_type_inc":"SimultaneousRecordingsTable","doc":"A table for grouping different intracellular recordings from the IntracellularRecordingsTable table together that were recorded simultaneously from different electrodes","quantity":"?"},{"name":"sequential_recordings","neurodata_type_inc":"SequentialRecordingsTable","doc":"A table for grouping different sequential recordings from the SimultaneousRecordingsTable table together. This is typically used to group together sequential recordings where the a sequence of stimuli of the same type with varying parameters have been presented in a sequence.","quantity":"?"},{"name":"repetitions","neurodata_type_inc":"RepetitionsTable","doc":"A table for grouping different sequential intracellular recordings together. With each SequentialRecording typically representing a particular type of stimulus, the RepetitionsTable table is typically used to group sets of stimuli applied in sequence.","quantity":"?"},{"name":"experimental_conditions","neurodata_type_inc":"ExperimentalConditionsTable","doc":"A table for grouping different intracellular recording repetitions together that belong to the same experimental experimental_conditions.","quantity":"?"}]},{"name":"optogenetics","doc":"Metadata describing optogenetic stimuluation.","quantity":"?","groups":[{"neurodata_type_inc":"OptogeneticStimulusSite","doc":"An optogenetic stimulation site.","quantity":"*"}]},{"name":"optophysiology","doc":"Metadata related to optophysiology.","quantity":"?","groups":[{"neurodata_type_inc":"ImagingPlane","doc":"An imaging plane.","quantity":"*"}]}]},{"name":"intervals","doc":"Experimental intervals, whether that be logically distinct sub-experiments having a particular scientific goal, trials (see trials subgroup) during an experiment, or epochs (see epochs subgroup) deriving from analysis of data.","quantity":"?","groups":[{"name":"epochs","neurodata_type_inc":"TimeIntervals","doc":"Divisions in time marking experimental stages or sub-divisions of a single recording session.","quantity":"?"},{"name":"trials","neurodata_type_inc":"TimeIntervals","doc":"Repeated experimental events that have a logical grouping.","quantity":"?"},{"name":"invalid_times","neurodata_type_inc":"TimeIntervals","doc":"Time intervals that should be removed from analysis.","quantity":"?"},{"neurodata_type_inc":"TimeIntervals","doc":"Optional additional table(s) for describing other experimental time intervals.","quantity":"*"}]},{"name":"units","neurodata_type_inc":"Units","doc":"Data about sorted spike units.","quantity":"?"}]},{"neurodata_type_def":"LabMetaData","neurodata_type_inc":"NWBContainer","doc":"Lab-specific meta-data."},{"neurodata_type_def":"Subject","neurodata_type_inc":"NWBContainer","doc":"Information about the animal or person from which the data was measured.","datasets":[{"name":"age","dtype":"text","doc":"Age of subject. Can be supplied instead of 'date_of_birth'.","quantity":"?","attributes":[{"name":"reference","doc":"Age is with reference to this event. Can be 'birth' or 'gestational'. If reference is omitted, 'birth' is implied.","dtype":"text","required":false,"default_value":"birth"}]},{"name":"date_of_birth","dtype":"isodatetime","doc":"Date of birth of subject. Can be supplied instead of 'age'.","quantity":"?"},{"name":"description","dtype":"text","doc":"Description of subject and where subject came from (e.g., breeder, if animal).","quantity":"?"},{"name":"genotype","dtype":"text","doc":"Genetic strain. If absent, assume Wild Type (WT).","quantity":"?"},{"name":"sex","dtype":"text","doc":"Gender of subject.","quantity":"?"},{"name":"species","dtype":"text","doc":"Species of subject.","quantity":"?"},{"name":"strain","dtype":"text","doc":"Strain of subject.","quantity":"?"},{"name":"subject_id","dtype":"text","doc":"ID of animal/person used/participating in experiment (lab convention).","quantity":"?"},{"name":"weight","dtype":"text","doc":"Weight at time of experiment, at time of surgery and at other important times.","quantity":"?"}]}],"datasets":[{"neurodata_type_def":"ScratchData","neurodata_type_inc":"NWBData","doc":"Any one-off datasets","attributes":[{"name":"notes","doc":"Any notes the user has about the dataset being stored","dtype":"text"}]}]} \ No newline at end of file diff --git a/resources/spec/core/2.7.0/nwb.namespace.json b/resources/spec/core/2.7.0/nwb.namespace.json index 19cada18..1ebae4be 100644 --- a/resources/spec/core/2.7.0/nwb.namespace.json +++ b/resources/spec/core/2.7.0/nwb.namespace.json @@ -1 +1 @@ -{"namespaces":[{"name":"core","doc":"NWB namespace","author":["Andrew Tritt","Oliver Ruebel","Ryan Ly","Ben Dichter","Keith Godfrey","Jeff Teeters"],"contact":["ajtritt@lbl.gov","oruebel@lbl.gov","rly@lbl.gov","bdichter@lbl.gov","keithg@alleninstitute.org","jteeters@berkeley.edu"],"full_name":"NWB core","schema":[{"namespace":"hdmf-common"},{"doc":"This source module contains base data types used throughout the NWB data format.","source":"nwb.base.yaml","title":"Base data types"},{"doc":"This source module contains neurodata_types for device data.","source":"nwb.device.yaml","title":"Devices"},{"doc":"This source module contains neurodata_types for epoch data.","source":"nwb.epoch.yaml","title":"Epochs"},{"doc":"This source module contains neurodata_types for image data.","source":"nwb.image.yaml","title":"Image data"},{"doc":"Main NWB file specification.","source":"nwb.file.yaml","title":"NWB file"},{"doc":"Miscellaneous types.","source":"nwb.misc.yaml","title":"Miscellaneous neurodata_types."},{"doc":"This source module contains neurodata_types for behavior data.","source":"nwb.behavior.yaml","title":"Behavior"},{"doc":"This source module contains neurodata_types for extracellular electrophysiology data.","source":"nwb.ecephys.yaml","title":"Extracellular electrophysiology"},{"doc":"This source module contains neurodata_types for intracellular electrophysiology data.","source":"nwb.icephys.yaml","title":"Intracellular electrophysiology"},{"doc":"This source module contains neurodata_types for opto-genetics data.","source":"nwb.ogen.yaml","title":"Optogenetics"},{"doc":"This source module contains neurodata_types for optical physiology data.","source":"nwb.ophys.yaml","title":"Optical physiology"},{"doc":"This source module contains neurodata_type for retinotopy data.","source":"nwb.retinotopy.yaml","title":"Retinotopy"}],"version":"2.7.0"}]} \ No newline at end of file +{"namespaces":[{"name":"core","doc":"NWB namespace","author":["Andrew Tritt","Oliver Ruebel","Ryan Ly","Ben Dichter","Keith Godfrey","Jeff Teeters"],"contact":["ajtritt@lbl.gov","oruebel@lbl.gov","rly@lbl.gov","bdichter@lbl.gov","keithg@alleninstitute.org","jteeters@berkeley.edu"],"full_name":"NWB core","schema":[{"namespace":"hdmf-common"},{"source":"nwb.base"},{"source":"nwb.device"},{"source":"nwb.epoch"},{"source":"nwb.image"},{"source":"nwb.file"},{"source":"nwb.misc"},{"source":"nwb.behavior"},{"source":"nwb.ecephys"},{"source":"nwb.icephys"},{"source":"nwb.ogen"},{"source":"nwb.ophys"},{"source":"nwb.retinotopy"}],"version":"2.7.0"}]} \ No newline at end of file diff --git a/resources/spec/hdmf-common/1.8.0/namespace.json b/resources/spec/hdmf-common/1.8.0/namespace.json index 73a9a2c2..0b921a45 100644 --- a/resources/spec/hdmf-common/1.8.0/namespace.json +++ b/resources/spec/hdmf-common/1.8.0/namespace.json @@ -1 +1 @@ -{"namespaces":[{"name":"hdmf-common","doc":"Common data structures provided by HDMF","author":["Andrew Tritt","Oliver Ruebel","Ryan Ly","Ben Dichter"],"contact":["ajtritt@lbl.gov","oruebel@lbl.gov","rly@lbl.gov","bdichter@lbl.gov"],"full_name":"HDMF Common","schema":[{"doc":"base data types","source":"base.yaml","title":"Base data types"},{"doc":"data types for a column-based table","source":"table.yaml","title":"Table data types"},{"doc":"data types for different types of sparse matrices","source":"sparse.yaml","title":"Sparse data types"}],"version":"1.5.0"},{"name":"hdmf-experimental","doc":"Experimental data structures provided by HDMF. These are not guaranteed to be available in the future","author":["Andrew Tritt","Oliver Ruebel","Ryan Ly","Ben Dichter"],"contact":["ajtritt@lbl.gov","oruebel@lbl.gov","rly@lbl.gov","bdichter@lbl.gov"],"full_name":"HDMF Experimental","schema":[{"namespace":"hdmf-common"},{"doc":"Experimental data types","source":"experimental.yaml","title":"Experimental data types"},{"doc":"data types for storing references to web accessible resources","source":"resources.yaml","title":"Resource reference data types"}],"version":"0.1.0"}]} \ No newline at end of file +{"namespaces":[{"name":"hdmf-common","doc":"Common data structures provided by HDMF","author":["Andrew Tritt","Oliver Ruebel","Ryan Ly","Ben Dichter"],"contact":["ajtritt@lbl.gov","oruebel@lbl.gov","rly@lbl.gov","bdichter@lbl.gov"],"full_name":"HDMF Common","schema":[{"source":"base"},{"source":"table"},{"source":"sparse"}],"version":"1.8.0"}]} \ No newline at end of file diff --git a/resources/spec/hdmf-common/1.8.0/resources.json b/resources/spec/hdmf-common/1.8.0/resources.json deleted file mode 100644 index 846dcfa9..00000000 --- a/resources/spec/hdmf-common/1.8.0/resources.json +++ /dev/null @@ -1 +0,0 @@ -{"groups":[{"data_type_def":"ExternalResources","data_type_inc":"Container","doc":"A set of four tables for tracking external resource references in a file. NOTE: this data type is in beta testing and is subject to change in a later version.","datasets":[{"data_type_inc":"Data","name":"keys","doc":"A table for storing user terms that are used to refer to external resources.","dtype":[{"name":"key","dtype":"text","doc":"The user term that maps to one or more resources in the 'resources' table."}],"dims":["num_rows"],"shape":[null]},{"data_type_inc":"Data","name":"entities","doc":"A table for mapping user terms (i.e., keys) to resource entities.","dtype":[{"name":"keys_idx","dtype":"uint","doc":"The index to the key in the 'keys' table."},{"name":"resources_idx","dtype":"uint","doc":"The index into the 'resources' table"},{"name":"entity_id","dtype":"text","doc":"The unique identifier entity."},{"name":"entity_uri","dtype":"text","doc":"The URI for the entity this reference applies to. This can be an empty string."}],"dims":["num_rows"],"shape":[null]},{"data_type_inc":"Data","name":"resources","doc":"A table for mapping user terms (i.e., keys) to resource entities.","dtype":[{"name":"resource","dtype":"text","doc":"The name of the resource."},{"name":"resource_uri","dtype":"text","doc":"The URI for the resource. This can be an empty string."}],"dims":["num_rows"],"shape":[null]},{"data_type_inc":"Data","name":"objects","doc":"A table for identifying which objects in a file contain references to external resources.","dtype":[{"name":"object_id","dtype":"text","doc":"The UUID for the object."},{"name":"field","dtype":"text","doc":"The field of the object. This can be an empty string if the object is a dataset and the field is the dataset values."}],"dims":["num_rows"],"shape":[null]},{"data_type_inc":"Data","name":"object_keys","doc":"A table for identifying which objects use which keys.","dtype":[{"name":"objects_idx","dtype":"uint","doc":"The index to the 'objects' table for the object that holds the key."},{"name":"keys_idx","dtype":"uint","doc":"The index to the 'keys' table for the key."}],"dims":["num_rows"],"shape":[null]}]}]} \ No newline at end of file diff --git a/resources/spec/hdmf-common/1.8.0/experimental.json b/resources/spec/hdmf-experimental/0.5.0/experimental.json similarity index 100% rename from resources/spec/hdmf-common/1.8.0/experimental.json rename to resources/spec/hdmf-experimental/0.5.0/experimental.json diff --git a/resources/spec/hdmf-experimental/0.5.0/namespace.json b/resources/spec/hdmf-experimental/0.5.0/namespace.json new file mode 100644 index 00000000..26f1ac7d --- /dev/null +++ b/resources/spec/hdmf-experimental/0.5.0/namespace.json @@ -0,0 +1 @@ +{"namespaces":[{"name":"hdmf-experimental","doc":"Experimental data structures provided by HDMF. These are not guaranteed to be available in the future.","author":["Andrew Tritt","Oliver Ruebel","Ryan Ly","Ben Dichter","Matthew Avaylon"],"contact":["ajtritt@lbl.gov","oruebel@lbl.gov","rly@lbl.gov","bdichter@lbl.gov","mavaylon@lbl.gov"],"full_name":"HDMF Experimental","schema":[{"namespace":"hdmf-common"},{"source":"experimental"},{"source":"resources"}],"version":"0.5.0"}]} \ No newline at end of file diff --git a/resources/spec/hdmf-experimental/0.5.0/resources.json b/resources/spec/hdmf-experimental/0.5.0/resources.json new file mode 100644 index 00000000..0e7d71ff --- /dev/null +++ b/resources/spec/hdmf-experimental/0.5.0/resources.json @@ -0,0 +1 @@ +{"groups":[{"data_type_def":"HERD","data_type_inc":"Container","doc":"HDMF External Resources Data Structure. A set of six tables for tracking external resource references in a file or across multiple files.","datasets":[{"data_type_inc":"Data","name":"keys","doc":"A table for storing user terms that are used to refer to external resources.","dtype":[{"name":"key","dtype":"text","doc":"The user term that maps to one or more resources in the `resources` table, e.g., \"human\"."}],"dims":["num_rows"],"shape":[null]},{"data_type_inc":"Data","name":"files","doc":"A table for storing object ids of files used in external resources.","dtype":[{"name":"file_object_id","dtype":"text","doc":"The object id (UUID) of a file that contains objects that refers to external resources."}],"dims":["num_rows"],"shape":[null]},{"data_type_inc":"Data","name":"entities","doc":"A table for mapping user terms (i.e., keys) to resource entities.","dtype":[{"name":"entity_id","dtype":"text","doc":"The compact uniform resource identifier (CURIE) of the entity, in the form [prefix]:[unique local identifier], e.g., 'NCBI_TAXON:9606'."},{"name":"entity_uri","dtype":"text","doc":"The URI for the entity this reference applies to. This can be an empty string. e.g., https://www.ncbi.nlm.nih.gov/Taxonomy/Browser/wwwtax.cgi?mode=info&id=9606"}],"dims":["num_rows"],"shape":[null]},{"data_type_inc":"Data","name":"objects","doc":"A table for identifying which objects in a file contain references to external resources.","dtype":[{"name":"files_idx","dtype":"uint","doc":"The row index to the file in the `files` table containing the object."},{"name":"object_id","dtype":"text","doc":"The object id (UUID) of the object."},{"name":"object_type","dtype":"text","doc":"The data type of the object."},{"name":"relative_path","dtype":"text","doc":"The relative path from the data object with the `object_id` to the dataset or attribute with the value(s) that is associated with an external resource. This can be an empty string if the object is a dataset that contains the value(s) that is associated with an external resource."},{"name":"field","dtype":"text","doc":"The field within the compound data type using an external resource. This is used only if the dataset or attribute is a compound data type; otherwise this should be an empty string."}],"dims":["num_rows"],"shape":[null]},{"data_type_inc":"Data","name":"object_keys","doc":"A table for identifying which objects use which keys.","dtype":[{"name":"objects_idx","dtype":"uint","doc":"The row index to the object in the `objects` table that holds the key"},{"name":"keys_idx","dtype":"uint","doc":"The row index to the key in the `keys` table."}],"dims":["num_rows"],"shape":[null]},{"data_type_inc":"Data","name":"entity_keys","doc":"A table for identifying which keys use which entity.","dtype":[{"name":"entities_idx","dtype":"uint","doc":"The row index to the entity in the `entities` table."},{"name":"keys_idx","dtype":"uint","doc":"The row index to the key in the `keys` table."}],"dims":["num_rows"],"shape":[null]}]}]} \ No newline at end of file diff --git a/src/BaseIO.hpp b/src/BaseIO.hpp index cf867d18..15e1a718 100644 --- a/src/BaseIO.hpp +++ b/src/BaseIO.hpp @@ -45,6 +45,7 @@ class BaseDataType T_F32, ///< 32-bit floating point T_F64, ///< 64-bit floating point T_STR, ///< String + V_STR, ///< Variable length string }; /** @@ -217,6 +218,15 @@ class BaseIO virtual Status createStringDataSet(const std::string& path, const std::string& value) = 0; + /** + * @brief Creates a dataset that holds an array of string values. + * @param path The location in the file of the dataset. + * @param values The vector of string values of the dataset. + * @return The status of the dataset creation operation. + */ + virtual Status createStringDataSet( + const std::string& path, const std::vector& values) = 0; + /** * @brief Creates a dataset that holds an array of references to groups within * the file. diff --git a/src/Utils.hpp b/src/Utils.hpp index 039596c6..a9c5fedf 100644 --- a/src/Utils.hpp +++ b/src/Utils.hpp @@ -3,10 +3,13 @@ #include #include +#include #include #include #include +#include "boost/date_time/c_local_time_adjustor.hpp" + namespace AQNWB { /** @@ -27,17 +30,25 @@ inline std::string generateUuid() */ inline std::string getCurrentTime() { - // Get current time - auto currentTime = - std::chrono::system_clock::to_time_t(std::chrono::system_clock::now()); + // Set up boost time zone adjustment and time facet + using local_adj = + boost::date_time::c_local_adjustor; + boost::posix_time::time_facet* f = new boost::posix_time::time_facet(); + f->time_duration_format("%+%H:%M"); - // Convert to tm struct to extract date and time components - std::tm utcTime = *std::gmtime(¤tTime); + // get local time, utc time, and offset + auto now = boost::posix_time::microsec_clock::universal_time(); + auto utc_now = local_adj::utc_to_local(now); + boost::posix_time::time_duration td = utc_now - now; // Format the date and time in ISO 8601 format with the UTC offset - std::ostringstream oss; - oss << std::put_time(&utcTime, "%FT%T%z"); + std::ostringstream oss_offset; + oss_offset.imbue(std::locale(oss_offset.getloc(), f)); + oss_offset << td; + + std::string currentTime = to_iso_extended_string(utc_now); + currentTime += oss_offset.str(); - return oss.str(); + return currentTime; } } // namespace AQNWB diff --git a/src/hdf5/HDF5IO.cpp b/src/hdf5/HDF5IO.cpp index 3d098a32..f6ade19c 100644 --- a/src/hdf5/HDF5IO.cpp +++ b/src/hdf5/HDF5IO.cpp @@ -342,6 +342,26 @@ Status HDF5IO::createStringDataSet(const std::string& path, return Status::Success; } +Status HDF5IO::createStringDataSet(const std::string& path, + const std::vector& values) +{ + if (!opened) + return Status::Failure; + + std::vector cStrs; + cStrs.reserve(values.size()); + for (const auto& str : values) { + cStrs.push_back(str.c_str()); + } + + std::unique_ptr dataset; + dataset = std::unique_ptr(createDataSet( + BaseDataType::V_STR, SizeArray {values.size()}, SizeArray {1}, path)); + dataset->writeDataBlock(1, BaseDataType::V_STR, cStrs.data()); + + return Status::Success; +} + AQNWB::BaseRecordingData* HDF5IO::getDataSet(const std::string& path) { std::unique_ptr data; @@ -441,6 +461,9 @@ H5::DataType HDF5IO::getNativeType(BaseDataType type) case BaseDataType::Type::T_STR: return StrType(PredType::C_S1, type.typeSize); break; + case BaseDataType::Type::V_STR: + return StrType(PredType::C_S1, H5T_VARIABLE); + break; default: baseType = PredType::NATIVE_INT32; } @@ -489,6 +512,9 @@ H5::DataType HDF5IO::getH5Type(BaseDataType type) case BaseDataType::Type::T_STR: return StrType(PredType::C_S1, type.typeSize); break; + case BaseDataType::Type::V_STR: + return StrType(PredType::C_S1, H5T_VARIABLE); + break; default: return PredType::STD_I32LE; } diff --git a/src/hdf5/HDF5IO.hpp b/src/hdf5/HDF5IO.hpp index 06295260..e7bdc5d8 100644 --- a/src/hdf5/HDF5IO.hpp +++ b/src/hdf5/HDF5IO.hpp @@ -153,6 +153,15 @@ class HDF5IO : public BaseIO Status createStringDataSet(const std::string& path, const std::string& value) override; + /** + * @brief Creates a dataset that holds an array of string values. + * @param path The location in the file of the dataset. + * @param values The vector of string values of the dataset. + * @return The status of the dataset creation operation. + */ + Status createStringDataSet(const std::string& path, + const std::vector& values) override; + /** * @brief Creates a dataset that holds an array of references to groups within * the file. diff --git a/src/nwb/NWBFile.cpp b/src/nwb/NWBFile.cpp index 3da373bd..3a28f6eb 100644 --- a/src/nwb/NWBFile.cpp +++ b/src/nwb/NWBFile.cpp @@ -43,10 +43,8 @@ void NWBFile::finalize() Status NWBFile::createFileStructure() { - io->createAttribute("core", "/", "namespace"); - io->createAttribute("NWBFile", "/", "neurodata_type"); + io->createCommonNWBAttributes("/", "core", "NWBFile", ""); io->createAttribute(NWBVersion, "/", "nwb_version"); - io->createAttribute(identifierText, "/", "object_id"); io->createGroup("/acquisition"); io->createGroup("/analysis"); @@ -59,15 +57,18 @@ Status NWBFile::createFileStructure() io->createGroup("general/extracellular_ephys"); io->createGroup("/specifications"); + io->createReferenceAttribute("/specifications", "/", ".specloc"); cacheSpecifications("core/", NWBVersion); cacheSpecifications("hdmf-common/", HDMFVersion); + cacheSpecifications("hdmf-experimental/", HDMFExperimentalVersion); std::string time = getCurrentTime(); - io->createStringDataSet("/file_create_date", time); // TODO - change to array + std::vector timeVec = {time}; + io->createStringDataSet("/file_create_date", timeVec); io->createStringDataSet("/session_description", "a recording session"); io->createStringDataSet("/session_start_time", time); io->createStringDataSet("/timestamps_reference_time", time); - io->createStringDataSet("/identifier", "test-identifier"); + io->createStringDataSet("/identifier", identifierText); return Status::Success; } diff --git a/src/nwb/NWBFile.hpp b/src/nwb/NWBFile.hpp index cba9e50a..4609b4ee 100644 --- a/src/nwb/NWBFile.hpp +++ b/src/nwb/NWBFile.hpp @@ -67,6 +67,11 @@ class NWBFile */ const std::string HDMFVersion = "1.8.0"; + /** + * @brief Indicates the HDMF experimental version. + */ + const std::string HDMFExperimentalVersion = "0.5.0"; + protected: /** * @brief Creates the default file structure. diff --git a/tests/aq-nwb_test.cpp b/tests/aq-nwb_test.cpp index fb2268a5..edc7bda1 100644 --- a/tests/aq-nwb_test.cpp +++ b/tests/aq-nwb_test.cpp @@ -4,6 +4,7 @@ #include #include "BaseIO.hpp" +#include "Utils.hpp" #include "hdf5/HDF5IO.hpp" #include "nwb/NWBFile.hpp" #include "nwb/file/ElectrodeTable.hpp" @@ -76,18 +77,20 @@ TEST_CASE("writeAttributes", "[hdf5io]") TEST_CASE("saveNWBFile", "[nwb]") { - std::string filename = getTestFilePath("test_nwb_file.h5"); + std::string filename = getTestFilePath("test_nwb_file.nwb"); - NWB::NWBFile nwbfile("123", std::make_unique(filename)); + NWB::NWBFile nwbfile(generateUuid(), + std::make_unique(filename)); nwbfile.initialize(); nwbfile.finalize(); } TEST_CASE("startRecording", "[nwb]") { - std::string filename = getTestFilePath("test_recording.h5"); + std::string filename = getTestFilePath("test_recording.nwb"); - NWB::NWBFile nwbfile("123", std::make_unique(filename)); + NWB::NWBFile nwbfile(generateUuid(), + std::make_unique(filename)); nwbfile.initialize(); Status result = nwbfile.startRecording(); nwbfile.finalize();