Update documentation to use sphinx (#643)

* Update main function docstrings to work better with sphinx/rst formatting

Add "sections"
Specify input and output arguments
Reformat examples

* Add tools for creating rst pages for functions, classes and tutorials

* Add docs source

* Add .readthedocs.yaml

* Delete previous doc files

* Update export location for tutorial html files

* remove html files from tutorials/html

(files are now located in docs/source/_static/html/tutorials)

* Fix indentation in function docstrings

* Add proper titles and "open in MATLAB Online" badges to tutorial pages

* Update main page and add installation instructions

* Add view full page badge for tutorials

* Add hdmf_common types plus smaller changes

Fix: Make sure neurodata types specfied with "short names" in the documentation for properties of generated classes are properly linked to their corresponding class pages in the docs i.e (TimeSeries)
Add links to the nwb format specification for each type

* Delete docstring_processors.cpython-311.pyc

* Update neurdata class rst template

* Update docstring_processors.py

- Add docstring processors for adding role to class properties and change formatting for sphinx-autodetected class constructor from reference to constructor method to reference for class

* Update generator to add more information in class docstrings

* Rerun generateCore (update all classes)

* Update link target attributes in livescript htmls

* Fix previous commit - use correct target attribute value

* Update links for types to point to readthedocs

* Update livescript html files

* Add favicon, dynamic iframe height, youtube badge for tutorial

* Update .codespellrc

* Fix links in tutorials, add hdmf_experimental types

* Fix wrong/outdated links in livescript tutorials tutorials

* Update tutorial_config.json

* Update pages

add section from matnwb readme
add pages from nwb-overview
minor rearrangement

* Minor fixes

---------

Co-authored-by: Ben Dichter <[email protected]>

+file/+internal/getRequiredPropertyNames.m

@@ -0,0 +1,26 @@
+function requiredPropertyNames = getRequiredPropertyNames(classprops)
+% getRequiredPropertyNames - Get name of required properties from props info
+
+    allProperties = keys(classprops);
+    requiredPropertyNames = {};
+
+    for iProp = 1:length(allProperties)
+
+        propertyName = allProperties{iProp};
+        prop = classprops(propertyName);
+
+        isRequired = ischar(prop) || isa(prop, 'containers.Map') || isstruct(prop);
+        isPropertyRequired = false;
+        if isa(prop, 'file.interface.HasProps')
+            isPropertyRequired = false(size(prop));
+            for iSubProp = 1:length(prop)
+                p = prop(iSubProp);
+                isPropertyRequired(iSubProp) = p.required;
+            end
+        end
+
+        if isRequired || all(isPropertyRequired)
+            requiredPropertyNames = [requiredPropertyNames {propertyName}]; %#ok<AGROW>
+        end
+    end
+end

+file/+internal/mergeProps.m

@@ -0,0 +1,18 @@
+function allProps = mergeProps(props, superClassProps)
+% merge_props - Merge maps containing props info for class and it's superclasses
+
+    allPropsCell = [{props}, superClassProps];
+    allProps = containers.Map();
+
+    % Start from most remote ancestor and work towards current class.
+    % Important to go in this order because subclasses can override
+    % properties, and we need to keep the property definition for the superclass
+    % that is closest to the current class or the property definition for the 
+    % class itself in the final map
+    for i = numel(allPropsCell):-1:1
+        superPropNames = allPropsCell{i}.keys;
+        for jProp = 1:numel(superPropNames)
+            allProps(superPropNames{jProp}) = allPropsCell{i}(superPropNames{jProp});
+        end
+    end
+end

+file/fillClass.m

@@ -1,4 +1,4 @@
-function template = fillClass(name, namespace, processed, classprops, inherited)
+function template = fillClass(name, namespace, processed, classprops, inherited, superClassProps)
     %name is the name of the scheme
     %namespace is the namespace context for this class
 
@@ -87,7 +87,20 @@
     %% return classfile string
     classDefinitionHeader = [...
         'classdef ' name ' < ' depnm ' & ' classTag newline... %header, dependencies
-        '% ' upper(name) ' ' class.doc]; %name, docstr
+        '% ' upper(name) ' - ' class.doc]; %name, docstr
+
+    allClassProps = file.internal.mergeProps(classprops, superClassProps);
+    allRequiredPropertyNames = file.internal.getRequiredPropertyNames(allClassProps);
+    if isempty(allRequiredPropertyNames)
+        allRequiredPropertyNames = {'None'};
+    end
+
+    % Add list of required properties in class docstring
+    classDefinitionHeader = [classDefinitionHeader, newline...
+        '%', newline, ...
+        '% Required Properties:', newline, ...
+        sprintf('%%  %s', strjoin(allRequiredPropertyNames, ', '))];
+
     hiddenAndReadonly = intersect(hidden, readonly);
     hidden = setdiff(hidden, hiddenAndReadonly);
     readonly = setdiff(readonly, hiddenAndReadonly);
@@ -126,7 +139,8 @@
         depnm,...
         defaults,... %all defaults, regardless of inheritance
         classprops,...
-        namespace);
+        namespace, ...
+        superClassProps);
     setterFcns = file.fillSetters(setdiff(nonInherited, union(readonly, hiddenAndReadonly)));
     validatorFcns = file.fillValidators(allProperties, classprops, namespace, namespace.getFullClassName(name), inherited);
     exporterFcns = file.fillExport(nonInherited, class, depnm);

+file/fillConstructor.m

@@ -1,6 +1,11 @@
-function functionString = fillConstructor(name, parentname, defaults, props, namespace)
+function functionString = fillConstructor(name, parentname, defaults, props, namespace, superClassProps)
     caps = upper(name);
-    functionBody = ['% ' caps ' Constructor for ' name];
+    functionBody = ['% ' caps ' - Constructor for ' name];
+
+    docString = fillConstructorDocString(name, props, namespace, superClassProps);
+    if ~isempty(docString)
+        functionBody = [functionBody newline() docString];
+    end
 
     bodyString = fillBody(parentname, defaults, props, namespace);
     if ~isempty(bodyString)
@@ -23,7 +28,6 @@
         ['function obj = ' name '(varargin)']...
         file.addSpaces(functionBody, 4)...
         'end'}, newline());
-
 end
 
 function bodystr = fillBody(parentName, defaults, props, namespace)
@@ -198,4 +202,116 @@
         '    types.util.dynamictable.checkConfig(obj);', ...
         'end',...
         }, newline);
+end
+
+function docString = fillConstructorDocString(name, props, namespace, superClassProps)
+
+    classVarName = name; classVarName(1) = lower(classVarName(1));
+    fullClassName = sprintf('types.%s.%s', namespace.name, name);
+    fullClassNameUpper = sprintf('types.%s.%s', namespace.name, upper(name));
+
+    props = file.internal.mergeProps(props, superClassProps);
+    names = props.keys();
+
+    docString = [...
+        "%", ...
+        "% Syntax:", ...
+        sprintf("%%  %s = %s() creates a %s object with unset property values.", classVarName, fullClassNameUpper, name), ...
+        "%", ...
+        ];
+
+    if ~isempty(names)
+        docString = [docString, ...
+            sprintf("%%  %s = %s(Name, Value) creates a %s object where one or more property values are specified using name-value pairs.", classVarName, fullClassNameUpper, name), ...
+            "%", ...
+            "% Input Arguments (Name-Value Arguments):", ...
+        ];
+    end
+
+    for i = 1:numel(names)
+        propName = names{i};
+        thisProp = props(propName);
+        try
+            if isprop(thisProp, 'readonly') && thisProp.readonly
+                continue
+            end
+        catch 
+            % pass
+        end
+
+        valueType = getTypeStr(thisProp);
+        try
+            description = thisProp.doc;
+        catch
+            description = 'No description';
+        end
+
+        docString = [docString, ...
+            sprintf("%%  - %s (%s) - %s", propName, valueType, description), ...
+            "%"]; %#ok<AGROW>
+    end
+
+    docString = [docString, ...
+        "% Output Arguments:", ...
+        sprintf("%%  - %s (%s) - A %s object", classVarName, fullClassName, name), ...
+        ""
+    ];
+
+    docString = char( strjoin(docString, newline) );
+end
+
+% Todo: Mostly duplicate code from file.fillProps. Should consolidate
+function typeStr = getTypeStr(prop)
+    if ischar(prop)
+        typeStr = prop;
+    elseif isstruct(prop)
+        columnNames = fieldnames(prop);
+        columnDocStr = cell(size(columnNames));
+        for i=1:length(columnNames)
+            name = columnNames{i};
+            columnDocStr{i} = getTypeStr(prop.(name));
+        end
+        typeStr = ['Table with columns: (', strjoin(columnDocStr, ', '), ')'];
+    elseif isa(prop, 'file.Attribute')
+        if isa(prop.dtype, 'containers.Map')
+            assertValidRefType(prop.dtype('reftype'))
+            typeStr = sprintf('%s reference to %s', capitalize(prop.dtype('reftype')), prop.dtype('target_type'));
+        else
+            typeStr = prop.dtype;
+        end
+    elseif isa(prop, 'containers.Map')
+        assertValidRefType(prop('reftype'))
+        typeStr = sprintf('%s reference to %s', capitalize(prop('reftype')), prop('target_type'));
+    elseif isa(prop, 'file.interface.HasProps')
+        typeStrCell = cell(size(prop));
+        for iProp = 1:length(typeStrCell)
+            anonProp = prop(iProp);
+            if isa(anonProp, 'file.Dataset') && isempty(anonProp.type)
+                typeStrCell{iProp} = getTypeStr(anonProp.dtype);
+            elseif isempty(anonProp.type)
+                typeStrCell{iProp} = 'types.untyped.Set';
+            else
+                typeStrCell{iProp} = anonProp.type;
+            end
+        end
+        typeStr = strjoin(typeStrCell, '|');
+    else
+        typeStr = prop.type;
+    end
+end
+
+function assertValidRefType(referenceType)
+    arguments
+        referenceType (1,1) string
+    end
+    assert( ismember(referenceType, ["region", "object"]), ...
+        'NWB:ClassGenerator:InvalidRefType', ...
+        'Invalid reftype found while filling description for class properties.')
+end
+
+function word = capitalize(word)
+    arguments
+        word (1,:) char
+    end
+    word(1) = upper(word(1));
 end

+file/writeNamespace.m

@@ -15,11 +15,16 @@ function writeNamespace(namespaceName, saveDir)
     [processed, classprops, inherited] = file.processClass(className, Namespace, pregenerated);
 
     if ~isempty(processed)
+        superClassProps = cell(1, numel(processed)-1);
+        for iSuper = 2:numel(processed)
+            [~, superClassProps{iSuper-1}, ~] =  file.processClass(processed(iSuper).type, Namespace, pregenerated);
+        end
+
         fid = fopen(fullfile(classFileDir, [className '.m']), 'W');
         % Create cleanup object to close to file in case the write operation fails.
         fileCleanupObj = onCleanup(@(id) fclose(fid));
         fwrite(fid, file.fillClass(className, Namespace, processed, ...
-            classprops, inherited), 'char');
+            classprops, inherited, superClassProps), 'char');
     else
         % pass
     end

+types/+core/AbstractFeatureSeries.m

@@ -1,5 +1,8 @@
 classdef AbstractFeatureSeries < types.core.TimeSeries & types.untyped.GroupClass
-% ABSTRACTFEATURESERIES Abstract features, such as quantitative descriptions of sensory stimuli. The TimeSeries::data field is a 2D array, storing those features (e.g., for visual grating stimulus this might be orientation, spatial frequency and contrast). Null stimuli (eg, uniform gray) can be marked as being an independent feature (eg, 1.0 for gray, 0.0 for actual stimulus) or by storing NaNs for feature values, or through use of the TimeSeries::control fields. A set of features is considered to persist until the next set of features is defined. The final set of features stored should be the null set. This is useful when storing the raw stimulus is impractical.
+% ABSTRACTFEATURESERIES - Abstract features, such as quantitative descriptions of sensory stimuli. The TimeSeries::data field is a 2D array, storing those features (e.g., for visual grating stimulus this might be orientation, spatial frequency and contrast). Null stimuli (eg, uniform gray) can be marked as being an independent feature (eg, 1.0 for gray, 0.0 for actual stimulus) or by storing NaNs for feature values, or through use of the TimeSeries::control fields. A set of features is considered to persist until the next set of features is defined. The final set of features stored should be the null set. This is useful when storing the raw stimulus is impractical.
+%
+% Required Properties:
+%  data, features
 
 
 % REQUIRED PROPERTIES
@@ -13,7 +16,47 @@
 
 methods
     function obj = AbstractFeatureSeries(varargin)
-        % ABSTRACTFEATURESERIES Constructor for AbstractFeatureSeries
+        % ABSTRACTFEATURESERIES - Constructor for AbstractFeatureSeries
+        %
+        % Syntax:
+        %  abstractFeatureSeries = types.core.ABSTRACTFEATURESERIES() creates a AbstractFeatureSeries object with unset property values.
+        %
+        %  abstractFeatureSeries = types.core.ABSTRACTFEATURESERIES(Name, Value) creates a AbstractFeatureSeries object where one or more property values are specified using name-value pairs.
+        %
+        % Input Arguments (Name-Value Arguments):
+        %  - comments (char) - Human-readable comments about the TimeSeries. This second descriptive field can be used to store additional information, or descriptive information if the primary description field is populated with a computer-readable string.
+        %
+        %  - control (uint8) - Numerical labels that apply to each time point in data for the purpose of querying and slicing data by these values. If present, the length of this array should be the same size as the first dimension of data.
+        %
+        %  - control_description (char) - Description of each control value. Must be present if control is present. If present, control_description[0] should describe time points where control == 0.
+        %
+        %  - data (numeric) - Values of each feature at each time.
+        %
+        %  - data_continuity (char) - Optionally describe the continuity of the data. Can be "continuous", "instantaneous", or "step". For example, a voltage trace would be "continuous", because samples are recorded from a continuous process. An array of lick times would be "instantaneous", because the data represents distinct moments in time. Times of image presentations would be "step" because the picture remains the same until the next timepoint. This field is optional, but is useful in providing information about the underlying data. It may inform the way this data is interpreted, the way it is visualized, and what analysis methods are applicable.
+        %
+        %  - data_conversion (single) - Scalar to multiply each element in data to convert it to the specified 'unit'. If the data are stored in acquisition system units or other units that require a conversion to be interpretable, multiply the data by 'conversion' to convert the data to the specified 'unit'. e.g. if the data acquisition system stores values in this object as signed 16-bit integers (int16 range -32,768 to 32,767) that correspond to a 5V range (-2.5V to 2.5V), and the data acquisition system gain is 8000X, then the 'conversion' multiplier to get from raw data acquisition values to recorded volts is 2.5/32768/8000 = 9.5367e-9.
+        %
+        %  - data_offset (single) - Scalar to add to the data after scaling by 'conversion' to finalize its coercion to the specified 'unit'. Two common examples of this include (a) data stored in an unsigned type that requires a shift after scaling to re-center the data, and (b) specialized recording devices that naturally cause a scalar offset with respect to the true units.
+        %
+        %  - data_resolution (single) - Smallest meaningful difference between values in data, stored in the specified by unit, e.g., the change in value of the least significant bit, or a larger number if signal noise is known to be present. If unknown, use -1.0.
+        %
+        %  - data_unit (char) - Since there can be different units for different features, store the units in 'feature_units'. The default value for this attribute is "see 'feature_units'".
+        %
+        %  - description (char) - Description of the time series.
+        %
+        %  - feature_units (char) - Units of each feature.
+        %
+        %  - features (char) - Description of the features represented in TimeSeries::data.
+        %
+        %  - starting_time (double) - Timestamp of the first sample in seconds. When timestamps are uniformly spaced, the timestamp of the first sample can be specified and all subsequent ones calculated from the sampling rate attribute.
+        %
+        %  - starting_time_rate (single) - Sampling rate, in Hz.
+        %
+        %  - timestamps (double) - Timestamps for samples stored in data, in seconds, relative to the common experiment master-clock stored in NWBFile.timestamps_reference_time.
+        %
+        % Output Arguments:
+        %  - abstractFeatureSeries (types.core.AbstractFeatureSeries) - A AbstractFeatureSeries object
+
         varargin = [{'data_unit' 'see `feature_units`'} varargin];
         obj = [email protected](varargin{:});
 

+types/+core/AnnotationSeries.m

@@ -1,11 +1,46 @@
 classdef AnnotationSeries < types.core.TimeSeries & types.untyped.GroupClass
-% ANNOTATIONSERIES Stores user annotations made during an experiment. The data[] field stores a text array, and timestamps are stored for each annotation (ie, interval=1). This is largely an alias to a standard TimeSeries storing a text array but that is identifiable as storing annotations in a machine-readable way.
+% ANNOTATIONSERIES - Stores user annotations made during an experiment. The data[] field stores a text array, and timestamps are stored for each annotation (ie, interval=1). This is largely an alias to a standard TimeSeries storing a text array but that is identifiable as storing annotations in a machine-readable way.
+%
+% Required Properties:
+%  data
 
 
 
 methods
     function obj = AnnotationSeries(varargin)
-        % ANNOTATIONSERIES Constructor for AnnotationSeries
+        % ANNOTATIONSERIES - Constructor for AnnotationSeries
+        %
+        % Syntax:
+        %  annotationSeries = types.core.ANNOTATIONSERIES() creates a AnnotationSeries object with unset property values.
+        %
+        %  annotationSeries = types.core.ANNOTATIONSERIES(Name, Value) creates a AnnotationSeries object where one or more property values are specified using name-value pairs.
+        %
+        % Input Arguments (Name-Value Arguments):
+        %  - comments (char) - Human-readable comments about the TimeSeries. This second descriptive field can be used to store additional information, or descriptive information if the primary description field is populated with a computer-readable string.
+        %
+        %  - control (uint8) - Numerical labels that apply to each time point in data for the purpose of querying and slicing data by these values. If present, the length of this array should be the same size as the first dimension of data.
+        %
+        %  - control_description (char) - Description of each control value. Must be present if control is present. If present, control_description[0] should describe time points where control == 0.
+        %
+        %  - data (char) - Annotations made during an experiment.
+        %
+        %  - data_continuity (char) - Optionally describe the continuity of the data. Can be "continuous", "instantaneous", or "step". For example, a voltage trace would be "continuous", because samples are recorded from a continuous process. An array of lick times would be "instantaneous", because the data represents distinct moments in time. Times of image presentations would be "step" because the picture remains the same until the next timepoint. This field is optional, but is useful in providing information about the underlying data. It may inform the way this data is interpreted, the way it is visualized, and what analysis methods are applicable.
+        %
+        %  - data_conversion (single) - Scalar to multiply each element in data to convert it to the specified 'unit'. If the data are stored in acquisition system units or other units that require a conversion to be interpretable, multiply the data by 'conversion' to convert the data to the specified 'unit'. e.g. if the data acquisition system stores values in this object as signed 16-bit integers (int16 range -32,768 to 32,767) that correspond to a 5V range (-2.5V to 2.5V), and the data acquisition system gain is 8000X, then the 'conversion' multiplier to get from raw data acquisition values to recorded volts is 2.5/32768/8000 = 9.5367e-9.
+        %
+        %  - data_offset (single) - Scalar to add to the data after scaling by 'conversion' to finalize its coercion to the specified 'unit'. Two common examples of this include (a) data stored in an unsigned type that requires a shift after scaling to re-center the data, and (b) specialized recording devices that naturally cause a scalar offset with respect to the true units.
+        %
+        %  - description (char) - Description of the time series.
+        %
+        %  - starting_time (double) - Timestamp of the first sample in seconds. When timestamps are uniformly spaced, the timestamp of the first sample can be specified and all subsequent ones calculated from the sampling rate attribute.
+        %
+        %  - starting_time_rate (single) - Sampling rate, in Hz.
+        %
+        %  - timestamps (double) - Timestamps for samples stored in data, in seconds, relative to the common experiment master-clock stored in NWBFile.timestamps_reference_time.
+        %
+        % Output Arguments:
+        %  - annotationSeries (types.core.AnnotationSeries) - A AnnotationSeries object
+
         varargin = [{'data_resolution' types.util.correctType(-1, 'single') 'data_unit' 'n/a'} varargin];
         obj = [email protected](varargin{:});