📝 Update FROM base from default to blank

bin/build

@@ -1,11 +1,19 @@
 #!/bin/bash
 
 # link references
-CPAC_DIRECTORY=$(python -c "import os, CPAC; print(os.path.abspath(os.path.dirname(CPAC.__file__)))")
-ln ${CPAC_DIRECTORY}/resources/configs/1.7-1.8-nesting-mappings.yml docs/_sources/references/1.7-1.8-nesting-mappings.yml || true
-ln ${CPAC_DIRECTORY}/resources/configs/1.7-1.8-deprecations.yml docs/_sources/references/1.7-1.8-deprecations.yml || true
-ln ${CPAC_DIRECTORY}/resources/configs/pipeline_config_default.yml docs/_sources/references/default_pipeline.yml || true
-ln ${CPAC_DIRECTORY}/resources/configs/group_config_template.yml docs/_sources/references/group_config_template.yml || true
+CPAC_DIRECTORY=$(python -c "from importlib import resources; print(str(resources.files('CPAC')))")
+ln "${CPAC_DIRECTORY}/resources/configs/1.7-1.8-nesting-mappings.yml" docs/_sources/references/1.7-1.8-nesting-mappings.yml || true
+ln "${CPAC_DIRECTORY}/resources/configs/1.7-1.8-deprecations.yml" docs/_sources/references/1.7-1.8-deprecations.yml || true
+shopt -s globstar nullglob
+for PRECONFIG in "${CPAC_DIRECTORY}"/resources/configs/pipeline_config_*.yml
+do
+    FILENAME=$(basename "$PRECONFIG")
+    CONFIG_NAME="${FILENAME#pipeline_config_}"
+    CONFIG_NAME="${CONFIG_NAME%.yml}"
+    echo "ln \"${PRECONFIG}\" \"docs/_sources/references/${CONFIG_NAME}_pipeline.yml\" || true"
+    ln "${PRECONFIG}" "docs/_sources/references/${CONFIG_NAME}_pipeline.yml" || true
+done
+ln "${CPAC_DIRECTORY}/resources/configs/group_config_template.yml" docs/_sources/references/group_config_template.yml || true
 
 # build docs
-sphinx-build -b html docs/_sources docs/$1
+sphinx-build -b html docs/_sources "docs/$1"

docs/_sources/conf.py

@@ -13,6 +13,7 @@
 # serve to show the default.
 
 import os
+from pathlib import Path
 import re
 import sys
 from typing import Any
@@ -28,6 +29,7 @@
 from nipype import __version__ as _nipype_version
 import semver
 from pybtex.plugin import register_plugin
+import yaml
 
 sys.path.append(os.path.dirname(__file__))
 
@@ -139,7 +141,29 @@ def yaml_to_rst(path):
     open(path, 'w').write(''.join(lines))
 
 
+def literal_include_config_page(filepath: Path) -> None:
+    """Create a page with a literalinclude of a config file."""
+    if not filepath.exists():
+        return
+    with filepath.open("r", encoding="utf8") as _f:
+        config: dict[str, Any] = yaml.safe_load(_f)
+    title: str = config["pipeline_setup"]["pipeline_name"]
+    with (Path("user/pipelines") /
+          filepath.name.replace("_pipeline.yml", ".rst")
+          ).open("w", encoding="utf8") as _f:
+        _f.write(f""".. title:: {title} pipeline configuration YAML file
+
+.. literalinclude:: /{filepath}
+   :language: YAML
+""")
+    return
+
+
 yaml_to_rst('references/1.7-1.8-deprecations.yml')
+for ref in Path("references").iterdir():
+    if str(ref).endswith("_pipeline.yml"):
+        # create literalinclude page
+        literal_include_config_page(ref)
 
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the

docs/_sources/developer/utils.rst

@@ -18,7 +18,15 @@ BIDS Data Configuration
 Configuration
 ^^^^^^^^^^^^^
 
-.. automodule:: CPAC.utils.configuration
+.. automodule:: CPAC.utils.configuration.configuration
+    :members:
+    :undoc-members:
+
+.. automodule:: CPAC.utils.configuration.diff
+    :members:
+    :undoc-members:
+
+.. automodule:: CPAC.utils.configuration.yaml_template
     :members:
     :undoc-members:
 

docs/_sources/developer/workflows/cpac_pipeline.rst

@@ -7,7 +7,7 @@ C-PAC Pipeline Construction
 Configuration Object
 ====================
 
-The 'c' variable in cpac_pipeline.py is a `Configuration object <https://github.com/FCP-INDI/C-PAC/blob/main/CPAC/utils/configuration.py>`_, which is an abstract representation used to store the options for the pipeline configuration YAML after it is read in by C-PAC.  Each attribute of the Configuration object is a key from the YAML file, with values mirroring the YAML.
+The 'c' variable in cpac_pipeline.py is a :py:class:`~CPAC.utils.configuration.configuration.Configuration` object, which is an abstract representation used to store the options for the pipeline configuration YAML after it is read in by C-PAC.  Each attribute of the Configuration object is a key from the YAML file, with values mirroring the YAML.
 
 Strategy Object
 ===============

docs/_sources/user/pipelines/pipeline_config.rst

@@ -51,15 +51,18 @@ If you want to base a pipeline on another pipeline configuration YAML file, you
 
    FROM: /path/to/pipeline.yml
 
-in your pipeline configuration file. You can use the name of a :doc:`preconfigured pipeline </user/pipelines/preconfig>` instead of a filepath if you want to base a configuration file on a preconfigured pipeline. If ``FROM`` is not specified, the pipeline will be based on :doc:`the default pipeline </user/pipelines/default>`.
+in your pipeline configuration file. You can use the name of a :doc:`preconfigured pipeline </user/pipelines/preconfig>` instead of a filepath if you want to base a configuration file on a preconfigured pipeline. If ``FROM`` is not specified, the pipeline will be based on :doc:`the blank pipeline </user/pipelines/blank>`.
+
+.. versionchanged:: 1.8.5
+   From version 1.8.0 to version 1.8.5, unspecified keys were based on the default configuration rather than the blank preconfiguration.
 
 C-PAC will include all expected keys from the pipeline file specified in ``FROM`` (or the default pipeline if none is specified). Any keys specified in a pipeline configuration file will take precedence over the same key in the ``FROM`` base configuration, but all omitted keys will retain their values from the ``FROM`` base configuration.
 
 From terminal, you can quickly generate a default pipeline configuration YAML file template in the directory you are in::
 
    cpac utils pipe-config new-template
 
-You can then edit the file as needed. For values that you want to leave at the default, you can either leave the key as-is, or you can remove the key, and C-PAC will automatically use value from the default pipeline configuration (or from the pipeline specified in ``FROM``).
+You can then edit the file as needed. For values that you want to leave at the default, you can either leave the key as-is, or you can add the line ``FROM: default``, remove the key, and C-PAC will automatically use value from the default pipeline configuration (or from the pipeline specified in ``FROM``).
 
 If you want to run the analysis from terminal::
 

docs/_sources/user/pipelines/preconfig.rst

@@ -7,7 +7,7 @@ human
 default: The Default Pipeline
 -----------------------------
 
-Pipeline Configuration YAML: `https://github.com/FCP-INDI/C-PAC/blob/main/CPAC/resources/configs/pipeline_config_default.yml <https://github.com/FCP-INDI/C-PAC/blob/main/CPAC/resources/configs/pipeline_config_default.yml>`_
+:doc:`Pipeline Configuration YAML </user/pipelines/default>`
 
 .. note::
 
@@ -54,14 +54,16 @@ References
 abcd-options
 ------------
 
+:doc:`Pipeline Configuration YAML </user/pipelines/abcd-options>`
+
 .. warning::
 
    :doc:`/user/known-issues/FCP-INDI/C-PAC/2104`
 
 anat-only: Default with Anatomical Preprocessing Only
 -----------------------------------------------------
 
-Pipeline Configuration YAML: `https://github.com/FCP-INDI/C-PAC/blob/main/CPAC/resources/configs/pipeline_config_anat-only.yml <https://github.com/FCP-INDI/C-PAC/blob/main/CPAC/resources/configs/pipeline_config_anat-only.yml>`_
+:doc:`Pipeline Configuration YAML </user/pipelines/anat-only>`
 
 Based on the preprocessing decisions of the default pipeline, this preconfiguration allows you to immediately kick off a run with only anatomical preprocessing selected. This includes:
 
@@ -72,7 +74,7 @@ Based on the preprocessing decisions of the default pipeline, this preconfigurat
 preproc: Default without Derivatives
 ------------------------------------
 
-Pipeline Configuration YAML: `https://github.com/FCP-INDI/C-PAC/blob/main/CPAC/resources/configs/pipeline_config_preproc.yml <https://github.com/FCP-INDI/C-PAC/blob/main/CPAC/resources/configs/pipeline_config_preproc.yml>`_
+:doc:`Pipeline Configuration YAML </user/pipelines/preproc>`
 
 Based on the preprocessing decisions of the default pipeline, this preconfiguration allows you to preprocess all of your data, without launching into calculation of outputs and data derivatives. This includes:
 
@@ -90,10 +92,10 @@ Functional:
 * Nuisance correction & filtering
 * Registration to template (via ANTs/ITK)
 
-fmriprep-options: fmriprep-Options Pipeline
+fmriprep-options: fMRIPrep options Pipeline
 -------------------------------------------
 
-Pipeline Configuration YAML: `https://github.com/FCP-INDI/C-PAC/blob/main/CPAC/resources/configs/pipeline_config_fmriprep-options.yml <https://github.com/FCP-INDI/C-PAC/blob/main/CPAC/resources/configs/pipeline_config_fmriprep-options.yml>`_
+:doc:`Pipeline Configuration YAML </user/pipelines/fmriprep-options>`
 
 This pipeline is designed to increase reproducibility with the preprocessing results of the fmriprep pipeline package\ :footcite:`fMRI16` produced by the `Poldrack Lab at Stanford University <https://poldracklab.stanford.edu/>`_.
 
@@ -111,7 +113,7 @@ References
 ndmg: Neurodata's 'ndmg-f' Pipeline
 -----------------------------------
 
-Pipeline Configuration YAML: `https://github.com/FCP-INDI/C-PAC/blob/main/CPAC/resources/configs/pipeline_config_ndmg.yml <https://github.com/FCP-INDI/C-PAC/blob/main/CPAC/resources/configs/pipeline_config_ndmg.yml>`_
+:doc:`Pipeline Configuration YAML </user/pipelines/ndmg>`
 
 This pipeline is the result of `Neurodata's <https://neurodata.io/>`_ study to converge upon the intersection of pipeline configuration decisions that maximizes discriminability between participants' data, drawing from the connectome graphs produced (labeled 'ndmg_graph' in the C-PAC output directory). This pipeline invokes a minimal set of preprocessing.
 
@@ -131,7 +133,7 @@ References
 rbc-options: ReproBrainChart Options Pipeline
 ---------------------------------------------
 
-Pipeline Configuration YAML: `https://github.com/FCP-INDI/C-PAC/blob/main/CPAC/resources/configs/pipeline_config_rbc-options.yml <https://github.com/FCP-INDI/C-PAC/blob/main/CPAC/resources/configs/pipeline_config_rbc-options.yml>`_
+:doc:`Pipeline Configuration YAML </user/pipelines/rbc-options>`
 
 RBC-options pipeline was built and integrated in C-PAC based on the Reproducible Brain Charts initiative, which aims to aggregate and harmonize phenotypic and neuroimage data to delineate node mechanisms regarding developmental basis of psychopathology in youth and yield reproducible growth charts of brain development\ :footcite:`Hoff21`.
 
@@ -146,7 +148,7 @@ non-human primate
 monkey: Default with Monkey Preprocessing 
 -----------------------------------------
 
-Pipeline Configuration YAML: `https://github.com/FCP-INDI/C-PAC/blob/main/CPAC/resources/configs/pipeline_config_monkey.yml <https://github.com/FCP-INDI/C-PAC/blob/main/CPAC/resources/configs/pipeline_config_monkey.yml>`_
+:doc:`Pipeline Configuration YAML </user/pipelines/monkey>`
 
 This pipeline is based on the work of Xu et al.\ :footcite:`Xu19` and nhp-ABCD-BIDS-pipeline.\ :footcite:`Stur20`
 
@@ -188,14 +190,14 @@ testing
 benchmark-ANTS: C-PAC Benchmark with ANTs Registration
 ------------------------------------------------------
 
-Pipeline Configuration YAML: `https://github.com/FCP-INDI/C-PAC/blob/main/CPAC/resources/configs/pipeline_config_benchmark-ANTS.yml <https://github.com/FCP-INDI/C-PAC/blob/main/CPAC/resources/configs/pipeline_config_benchmark-ANTS.yml>`_
+:doc:`Pipeline Configuration YAML </user/pipelines/benchmark-ANTS>`
 
 The benchmark pipeline has remained mostly unchanged since the project's inception, and is used at the end of each release cycle to ensure the results of C-PAC's key outputs have not changed. It is designed to test a wide range of pipeline options. This pipeline is based on registration-to-template using the ANTs/ITK toolset, as this decision impacts many other aspects of the pipeline further downstream.
 
 benchmark-FNIRT: C-PAC Benchmark with FSL FNIRT Registration
 ------------------------------------------------------------
 
-Pipeline Configuration YAML: `https://github.com/FCP-INDI/C-PAC/blob/main/CPAC/resources/configs/pipeline_config_benchmark-FNIRT.yml <https://github.com/FCP-INDI/C-PAC/blob/main/CPAC/resources/configs/pipeline_config_benchmark-FNIRT.yml>`_
+:doc:`Pipeline Configuration YAML </user/pipelines/benchmark-FNIRT>`
 
 The benchmark pipeline has remained mostly unchanged since the project's inception, and is used at the end of each release cycle to ensure the results of C-PAC's key outputs have not changed. It is designed to test a wide range of pipeline options. This pipeline is based on registration-to-template using the FSL FLIRT & FNIRT, as this decision impacts many other aspects of the pipeline further downstream.