Add documentation for the file formats

CHANGELOG.rst

@@ -1,6 +1,9 @@
 Changelog
 =========
 
+Version 3.1.1
+-------------
+- Add documentation for ROI Mask, Orientation and scalar file formats
 
 Version 3.1.0
 -------------

doc/source/atlas.rst

@@ -19,7 +19,7 @@ Volumetric datasets
 
 Volumetric datasets are stored in VoxelBrain in `NRRD <http://teem.sourceforge.net/nrrd/format.html>`_ format, which is basically a grid of values for each voxel + some metadata defining voxel size and location in the global atlas space.
 
-The values stored in volumetric datasets could be `scalar <https://bbpteam.epfl.ch/project/spaces/display/NRINF/Scalar+Value+Image>`_ (e.g., brain region ID, cell density); as well as vector (e.g., morphology `orientation field <https://bbpteam.epfl.ch/project/spaces/display/NRINF/Orientation+Field>`_).
+The values stored in volumetric datasets could be :ref:`Scalar Image File Format` (e.g., brain region ID, cell density); as well as vector (e.g., morphology orientation `OrientationField`_).
 
 For the atlas mentioned above, NRRDs for each volumetric dataset available could be fetched from `<http://voxels.nexus.apps.bbp.epfl.ch/api/analytics/atlas/releases/9B1F97DD-13B8-4FCF-B9B1-59E4EBE4B5D8/data>`_.
 
@@ -61,15 +61,17 @@ Alternatively, one can provide ``outer_value`` optional argument to return a stu
     >> vd.lookup(xyz, outer_value=-1)
 
 
-TODO: document other methods
+.. todo::
+
+   document other methods
 
 
 OrientationField
 ^^^^^^^^^^^^^^^^
 
 |name| provides this subclass of ``VoxelData`` for transparently converting quaternions stored in orientation fields to rotation matrices form.
 
-Thus, given a NRRD in the specific `format <https://bbpteam.epfl.ch/project/spaces/display/NRINF/Orientation+Field>`__, one can:
+Thus, given a NRRD in the :ref:`Orientation Field File Format`, one can:
 
 .. code-block:: python
 
@@ -83,7 +85,7 @@ ROIMask
 
 |name| provides this subclass of ``VoxelData`` for transparently loading masks and converting values from ``int8`` or ``uint8`` to ``bool``.
 
-Thus, given a NRRD in the specific `format <https://bbpteam.epfl.ch/project/spaces/pages/viewpage.action?pageId=27234876>`__, one can:
+Thus, given a NRRD in the specific :ref:`Mask Image for Region of Interest (ROI)`, one can:
 
 .. code-block:: python
 

doc/source/conf.py

@@ -12,13 +12,19 @@
 # All configuration values have a default; values that are commented out
 # serve to show the default.
 
+import os
+import sys
+
 import voxcell
 
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
 #sys.path.insert(0, os.path.abspath('.'))
 
+local_path = os.path.dirname(os.path.abspath(__file__))
+sys.path.insert(0, os.path.join(local_path, 'extensions'))
+
 # -- General configuration ------------------------------------------------
 
 # If your documentation needs a minimal Sphinx version, state it here.
@@ -29,14 +35,21 @@
 # ones.
 extensions = [
     'sphinx.ext.autodoc',
+    'sphinx.ext.autosummary',
+    'sphinx.ext.autosectionlabel',
     'sphinx.ext.graphviz',
     'sphinx.ext.ifconfig',
-    'sphinx.ext.viewcode',
-    'sphinx.ext.napoleon',
     'sphinx.ext.mathjax',
-    'sphinx.ext.autosummary',
+    'sphinx.ext.napoleon',
+    'sphinx.ext.todo',
+    'sphinx.ext.viewcode',
+    'bbp-table',
 ]
 
+# because we ..include CHANGELOG, the sections get included twice
+suppress_warnings = ['autosectionlabel.*',
+                     ]
+
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['']
 

doc/source/extensions/bbp-table/__init__.py

@@ -0,0 +1,140 @@
+from collections import namedtuple
+from docutils.parsers.rst import Directive, directives
+from docutils import nodes
+
+
+class bbp_table_value(nodes.Element):
+    pass
+
+
+class BBPTableValue(Directive):
+    has_content = True
+    required_arguments = 1
+    optional_arguments = 0
+    final_argument_whitespace = True
+    option_spec = {'type': directives.unchanged,
+                   'value': directives.unchanged,
+                   'comment': directives.unchanged
+                   }
+
+    def run(self):
+        resultnode = bbp_table_value()
+        self.options['field'] = self.arguments[0]
+        resultnode.options = self.options
+        return [resultnode]
+
+
+BBPTableSectionInfo = namedtuple('BBPTableSectionInfo',
+                                   'name, description, docname, targetid')
+
+
+class bbp_table_section(nodes.General, nodes.Element):
+    pass
+
+
+class BBPTableSection(Directive):
+    has_content = True
+    required_arguments = 1
+    optional_arguments = 0
+    final_argument_whitespace = True
+    option_spec = {'description': directives.unchanged,
+                   }
+
+    def _make_table(self, rows):
+        def create_table_row(row_cells):
+            row = nodes.row()
+            for cell in row_cells:
+                row += nodes.entry('', nodes.paragraph(text=cell))
+            return row
+
+        header = ('Field', 'Value Type', 'Suggested Value', 'Comments' )
+        colwidths = (1, 1, 2, 3)
+
+        assert len(header) == len(colwidths)
+        tgroup = nodes.tgroup(cols=len(header))
+        for c in colwidths:
+            tgroup += nodes.colspec(colwidth=c)
+        tgroup += nodes.thead('', create_table_row(header))
+        tbody = nodes.tbody()
+        tgroup += tbody
+        for row in rows:
+            tbody += create_table_row((row.options['field'],
+                                       row.options['type'],
+                                       row.options['value'],
+                                       row.options['comment']))
+
+        table = nodes.table('', tgroup)
+        return table
+
+    def run(self):
+        env = self.state.document.settings.env
+        if not hasattr(env, 'all_bbp_table_sections'):
+            env.all_bbp_table_sections = []
+
+        name = self.arguments[0]
+        description = self.options['description']
+        targetid = "bbp_tablesection-%d" % env.new_serialno('bbp_table_section')
+
+        node = nodes.Element()
+        self.state.nested_parse(self.content, self.content_offset, node)
+
+        section_info = BBPTableSectionInfo(name, description, env.docname, targetid)
+        env.all_bbp_table_sections.append(section_info)
+
+        children = []
+        for child in node:
+            if isinstance(child, bbp_table_value):
+                children.append(child)
+
+        resultnode = nodes.section(ids=[targetid])
+        resultnode += [nodes.title(text=name),
+                       nodes.paragraph(text=description),
+                       self._make_table(children),
+                       nodes.line()
+                       ]
+
+        return [resultnode]
+
+
+class bbp_table_section_index(nodes.Element):
+    pass
+
+
+class BBPTableSectionIndex(Directive):
+    '''create a place-holder for an index'''
+
+    def run(self):
+        return [bbp_table_section_index('')]
+
+
+def process_bbp_table_section_index(app, doctree, fromdocname):
+    env = app.builder.env
+
+    for node in doctree.traverse(bbp_table_section_index):
+        references = []
+        for section in env.all_bbp_table_sections:
+            ref = nodes.reference(section.name, section.name)
+            ref['refdocname'] = section.docname
+            ref['refuri'] = app.builder.get_relative_uri(
+                fromdocname, section.docname) + '#' + section.targetid
+
+            para = nodes.paragraph('', '', ref)
+            item = nodes.list_item('', para, nodes.paragraph(text=section.description))
+            references.append(item)
+
+        content = nodes.bullet_list('', *references)
+        node.replace_self(content)
+
+
+def setup(app):
+    app.add_node(bbp_table_section_index)
+    app.add_directive('bbp_table_section_index', BBPTableSectionIndex)
+
+    app.add_node(bbp_table_section)
+    app.add_directive('bbp_table_section', BBPTableSection)
+    app.add_config_value('bbp_table_section', {}, 'env')
+
+    app.add_node(bbp_table_value)
+    app.add_directive('bbp_table_value', BBPTableValue)
+
+    app.connect('doctree-resolved', process_bbp_table_section_index)

doc/source/formats.rst

@@ -0,0 +1,10 @@
+NRRD File Formats
+=================
+
+.. toctree::
+   :hidden:
+
+   mask
+   orientation
+   scalar
+

doc/source/index.rst

@@ -21,6 +21,7 @@ Contents
    aibs
    atlas
    extras
+   formats
    api
    changelog
 

doc/source/mask.rst

@@ -0,0 +1,71 @@
+Mask Image for Region of Interest (ROI)
+=======================================
+
+.. _mask_roi:
+
+For validating the circuits we would need to specify "regions of interest" for a given atlas (for example, pick a sphere within a layer to measure cell density). A mask image is generated for given specific shape and stored in NRRD format with the following metadata.
+
+.. bbp_table_section:: Specification
+    :description:
+     The meta data stored in the NRRD file should follow the specification below:
+
+    .. bbp_table_value:: dimension
+        :type: integer
+        :value: 3
+        :comment: dimension of the image is always 3 for volumetric data
+
+    .. bbp_table_value:: encoding
+        :type: string
+        :value: gzip
+        :comment: use 'gzip' for compressing the data
+
+    .. bbp_table_value:: endian
+        :type: string
+        :value: little
+        :comment: use 'little' for x86 machines
+
+    .. bbp_table_value:: kinds
+        :type: list of string
+        :value: ['domain', 'domain', 'domain']
+        :comment: use 'domain' to indicate that the axis is image axis
+
+    .. bbp_table_value:: sizes
+        :type: list of integer
+        :value: [{size_x}, {size_y}, {size_z}]
+        :comment: the size of the volume in x, y, z dimension
+
+    .. bbp_table_value:: space directions
+        :type: 3d matrix
+        :value:
+        :comment: A 3D matrix indicating the orientation of the image data, with the value indicating the spacing of the voxel
+
+    .. bbp_table_value:: space origin
+        :type: list of float
+        :value: [{origin_x}, {origin_y}, {origin_z}]
+        :comment: physical coordinates of the image origin
+
+    .. bbp_table_value:: space
+        :type: string
+        :value: {x_orientation}-{y_orientation}-{z_orientation}
+        :comment: the orientation of the image data, should be consistent with the space direction
+
+    .. bbp_table_value:: type
+        :type: string
+        :value: uchar
+        :comment: the type of the values stored in the voxels is unsigned char
+
+Example:
+
+::
+
+  {
+    u'space origin': ['-46.540000915527344', '-152.15999984741211', '-152'],
+    u'space directions': [['16', '0', '0'], ['0', '16', '0'], ['0', '0', '16']],
+    u'sizes': [308, 495, 464],
+    u'space': 'left-posterior-superior',
+    u'encoding': 'gzip',
+    u'endian': 'little',
+    u'kinds': ['domain', 'domain', 'domain'],
+    u'type': 'unsigned char',
+    u'dimension': 3
+  }

doc/source/orientation.rst

@@ -0,0 +1,70 @@
+Orientation Field File Format
+=============================
+
+
+The orientation field indicates the orientation of the morphology at each voxel position in the form of quaternion. The coefficients of the quaternion is encoded in the order of (w, x, y, z).Therefore, each voxel stores a 4d vector of float number in the NRRD file.
+
+.. bbp_table_section:: Specification
+    :description:
+     The meta data stored in the NRRD file should follow the specification below:
+
+    .. bbp_table_value:: dimension
+        :type: integer
+        :value: 4
+        :comment: dimension of the image is 4, with the 4th dimension indicating the quaternion 4d vector
+
+    .. bbp_table_value:: encoding
+        :type: string
+        :value: gzip
+        :comment: use 'gzip' for compressing the data
+
+    .. bbp_table_value:: endian
+        :type: string
+        :value: little
+        :comment: use 'little' for x86 machines
+
+    .. bbp_table_value:: kinds
+        :type: list of string
+        :value: ['quaternion', 'domain', 'domain', 'domain']
+        :comment: use 'quaternion' to indicate the type of the first data dimension, use 'domain' to indicate the other dimensions are image axis
+
+    .. bbp_table_value:: sizes
+        :type: list of integer
+        :value: [4, {size_x}, {size_y}, {size_z}]
+        :comment: the 1st element is the size of quaternion vector which is 4, together with the size of the voxels in x, y, z dimension
+
+    .. bbp_table_value:: space directions
+        :type: 3d matrix
+        :value: [u'none', {3d matrix}]
+        :comment: 'none' indicate the first dimension does not represent the image space. the second part is a 3D matrix indicating the orientation of the image data, with the value indicating the spacing of the voxel
+
+    .. bbp_table_value:: space origin
+        :type: list of float
+        :value: [{origin_x}, {origin_y}, {origin_z}]
+        :comment: physical coordinates of the image origin
+
+    .. bbp_table_value:: space
+        :type: string
+        :value: {x_orientation}-{y_orientation}-{z_orientation}
+        :comment: the orientation of the image data, should be consistent with the space direction
+
+    .. bbp_table_value:: type
+        :type: string
+        :value: float (default) or int8 (optional)
+        :comment: the default type of the values stored in the voxels is float. alternatively, the type of the values stored in the voxels can be signed 1-byte integer to minimize the file size
+
+Example:
+
+::
+
+  {
+    u'space origin': ['-46.540000915527344', '-152.15999984741211', '-152'],
+    u'space directions': [u'none', ['16', '0', '0'], ['0', '16', '0'], ['0', '0', '16']],
+    u'sizes': [4, 308, 495, 464],
+    u'space': 'left-posterior-superior',
+    u'encoding': 'gzip',
+    u'endian': 'little',
+    u'kinds': ['quaternion', 'domain', 'domain', 'domain'],
+    u'type': 'int8',
+    u'dimension': 4
+  }