refactor(#317): Better design

- Continue moving things over to the new PathSet() paradigm

- I MAY want to have the batch criteria own its own pathset; not sure if that's
  a good idea or not. Will revisit.

docs/src/tutorials/project/template_input_file.rst

@@ -165,7 +165,7 @@ Any of the following may be inserted:
                <param name="experiment/length" value="1234"/>
                <param name="experiment/random_seed" value="5678"/>
                <param name="experiment/param_file" value="/path/to/file"/>
-               <param name="experiment/n_robots" value="123"/>
+               <param name="experiment/n_agents" value="123"/>
                <param name="experiment/ticks_per_sec" value="5"/>
             </group>
             ...
@@ -189,7 +189,7 @@ Any of the following may be inserted:
                <param name="experiment/length" value="1234"/>
                <param name="experiment/random_seed" value="5678"/>
                <param name="experiment/param_file" value="/path/to/file"/>
-               <param name="experiment/n_robots" value="123"/>
+               <param name="experiment/n_agents" value="123"/>
                <param name="experiment/ticks_per_sec" value="5"/>
             </group>
             ...
@@ -224,7 +224,7 @@ Any of the following may be inserted:
                 <param name="experiment/length" value="1234"/>
                 <param name="experiment/random_seed" value="5678"/>
                 <param name="experiment/param_file" value="/path/to/file"/>
-                <param name="experiment/n_robots" value="123"/>
+                <param name="experiment/n_agents" value="123"/>
                 <param name="experiment/ticks_per_sec" value="5"/>
              </group>
              ...
@@ -247,7 +247,7 @@ Any of the following may be inserted:
                 <param name="experiment/length" value="1234"/>
                 <param name="experiment/random_seed" value="5678"/>
                 <param name="experiment/param_file" value="/path/to/file"/>
-                <param name="experiment/n_robots" value="123"/>
+                <param name="experiment/n_agents" value="123"/>
                 <param name="experiment/ticks_per_sec" value="5"/>
              </group>
              ...

sierra/core/__init__.py

@@ -7,5 +7,3 @@
 # 3rd party packages
 
 # Project packages
-from . import cmdline
-from . import config

sierra/core/batchroot.py

@@ -94,18 +94,18 @@ def to_path(self) -> pathlib.Path:
 
 class PathSet():
     def __init__(self, root: ExpRoot) -> None:
-        self.root = root
-        self.input_root = self.root.to_path() / "exp-inputs"
-        self.output_root = self.root.to_path() / "exp-outputs"
-        self.graph_root = self.root.to_path() / "graphs"
-        self.model_root = self.root.to_path() / "models"
-        self.stat_root = self.root.to_path() / "statistics"
+        self.input_root = root.to_path() / "exp-inputs"
+        self.output_root = root.to_path() / "exp-outputs"
+        self.graph_root = root.to_path() / "graphs"
+        self.model_root = root.to_path() / "models"
+        self.stat_root = root.to_path() / "statistics"
         self.stat_exec_root = self.stat_root.to_path() / "exec"
-        self.imagize_root = self.root.to_path() / "imagize"
-        self.video_root = self.root.to_path() / "videos"
-        self.stat_collate = self.stat_root.to_path() / "collated"
-        self.graph_collate = self.graph_root.to_path() / "collated"
-        self.scratch_root = self.root.to_path() / "scratch"
+        self.imagize_root = root.to_path() / "imagize"
+        self.video_root = root.to_path() / "videos"
+        self.stat_collate_root = self.stat_root.to_path() / "collated"
+        self.graph_collate_root = self.graph_root.to_path() / "collated"
+        self.scratch_root = root.to_path() / "scratch"
+        self.root = root.to_path()
 
 
 def from_cmdline(args: argparse.Namespace) -> PathSet:
@@ -128,7 +128,7 @@ def from_cmdline(args: argparse.Namespace) -> PathSet:
                     args.controller)
 
 
-def from_exp(sierra_rpath: str,
+def from_exp(sierra_root: str,
              project: str,
              batch_leaf: ExpRootLeaf,
              controller: str) -> PathSet:
@@ -150,7 +150,7 @@ def from_exp(sierra_rpath: str,
         controller: The name of the controller used.
 
     """
-    root = ExpRoot(sierra_rpath,
+    root = ExpRoot(sierra_root,
                    project,
                    controller,
                    batch_leaf)

sierra/core/experiment/bindings.py

@@ -135,7 +135,7 @@ class IExpRunShellCmdsGenerator(implements.Interface):
 
         cmdopts: Dictionary of parsed cmdline options.
 
-        n_robots: The configured # of robots for the experimental run.
+        n_agents: The configured # of robots for the experimental run.
 
         exp_num: The 0-based index of the experiment in the batch.
 
@@ -144,7 +144,7 @@ class IExpRunShellCmdsGenerator(implements.Interface):
     def __init__(self,
                  cmdopts: types.Cmdopts,
                  criteria: bc.BatchCriteria,
-                 n_robots: int,
+                 n_agents: int,
                  exp_num: int) -> None:
         raise NotImplementedError
 

sierra/core/experiment/spec.py

@@ -33,7 +33,7 @@ def __init__(self,
         self.mods = []
         self.is_compound = False
 
-        assert len(self.rms) == 0,\
+        assert len(self.rms) == 0, \
             "Batch criteria cannot remove XML tags"
 
         if self.chgs:
@@ -79,7 +79,7 @@ def __init__(self,
         self.is_compound = True
         self.mods = []
 
-        assert len(self.rms) == 0,\
+        assert len(self.rms) == 0, \
             "Batch criteria cannot remove XML tags"
 
         if self.chgs and self.adds:
@@ -125,12 +125,13 @@ class ExperimentSpec():
 
     def __init__(self,
                  criteria: bc.IConcreteBatchCriteria,
+                 batch_input_root: pathlib.Path,
                  exp_num: int,
                  cmdopts: types.Cmdopts) -> None:
         self.exp_num = exp_num
-        exp_name = criteria.gen_exp_names(cmdopts)[exp_num]
+        exp_name = criteria.gen_exp_names()[exp_num]
 
-        self.exp_input_root = pathlib.Path(cmdopts['batch_input_root'], exp_name)
+        self.exp_input_root = batch_input_root / exp_name
         self.exp_def_fpath = self.exp_input_root / config.kPickleLeaf
 
         self.logger = logging.getLogger(__name__)

sierra/core/exproot.py

@@ -0,0 +1,29 @@
+#
+# Copyright 2024 John Harwell, All rights reserved.
+#
+# SPDX-License Identifier: MIT
+#
+
+# Core packages
+import typing as tp
+
+# 3rd party packages
+
+# Project packages
+from sierra.core import batchroot
+
+
+class PathSet():
+    def __init__(self,
+                 batch: batchroot.PathSet,
+                 exp_name: str,
+                 exp0_name: tp.Optional[str] = None) -> None:
+        self.input_root = batch.input_root.to_path() / exp_name
+        self.output_root = batch.output_root.to_path() / exp_name
+        self.graph_root = batch.graph_root.to_path() / exp_name
+        self.model_root = batch.model_root.to_path() / exp_name
+        self.stat_root = batch.state_root.to_path() / exp_name
+
+        if exp0_name:
+            self.exp0_output_root = batch.output_root.to_path() / exp0_name
+            self.exp0_stat_root = batch.stat_root.to_path() / exp0_name

sierra/core/generators/exp_creator.py

@@ -21,7 +21,7 @@
 
 # Project packages
 from sierra.core.variables import batch_criteria as bc
-from sierra.core import config, utils, types, platform
+from sierra.core import config, utils, types, platform, batchroot
 import sierra.core.plugin_manager as pm
 from sierra.core.generators.exp_generators import BatchExpDefGenerator
 from sierra.core.experiment import bindings, definition
@@ -118,13 +118,13 @@ def from_def(self, exp_def: definition.XMLExpDef):
         if configurer.cmdfile_paradigm() == 'per-exp' and utils.path_exists(commands_fpath):
             commands_fpath.unlink()
 
-        n_robots = utils.get_n_robots(self.criteria.main_config,
+        n_agents = utils.get_n_agents(self.criteria.main_config,
                                       self.cmdopts,
                                       self.exp_input_root,
                                       exp_def)
         generator = platform.ExpRunShellCmdsGenerator(self.cmdopts,
                                                       self.criteria,
-                                                      n_robots,
+                                                      n_agents,
                                                       self.exp_num)
 
         # Create all experimental runs
@@ -229,21 +229,21 @@ def _update_cmds_file(self,
         pre_specs = cmds_generator.pre_run_cmds(for_host,
                                                 launch_stem_path,
                                                 run_num)
-        assert all(spec.shell for spec in pre_specs),\
+        assert all(spec.shell for spec in pre_specs), \
             "All pre-exp commands are run in a shell"
         pre_cmds = [spec.cmd for spec in pre_specs]
         self.logger.trace("Pre-experiment cmds: %s", pre_cmds)   # type: ignore
 
         exec_specs = cmds_generator.exec_run_cmds(for_host,
                                                   launch_stem_path,
                                                   run_num)
-        assert all(spec.shell for spec in exec_specs),\
+        assert all(spec.shell for spec in exec_specs), \
             "All exec-exp commands are run in a shell"
         exec_cmds = [spec.cmd for spec in exec_specs]
         self.logger.trace("Exec-experiment cmds: %s", exec_cmds)   # type: ignore
 
         post_specs = cmds_generator.post_run_cmds(for_host)
-        assert all(spec.shell for spec in post_specs),\
+        assert all(spec.shell for spec in post_specs), \
             "All post-exp commands are run in a shell"
         post_cmds = [spec.cmd for spec in post_specs]
         self.logger.trace("Post-experiment cmds: %s", post_cmds)   # type: ignore
@@ -298,11 +298,12 @@ class BatchExpCreator:
 
     def __init__(self,
                  criteria: bc.BatchCriteria,
-                 cmdopts: types.Cmdopts) -> None:
+                 cmdopts: types.Cmdopts,
+                 pathset: batchroot.PathSet) -> None:
 
         self.batch_config_template = pathlib.Path(cmdopts['template_input_file'])
-        self.batch_input_root = pathlib.Path(cmdopts['batch_input_root'])
-        self.batch_output_root = pathlib.Path(cmdopts['batch_output_root'])
+        self.batch_input_root = pathset.input_root
+        self.batch_output_root = pathset.output_root
         self.criteria = criteria
         self.cmdopts = cmdopts
         self.logger = logging.getLogger(__name__)
@@ -333,7 +334,7 @@ def create(self, generator: BatchExpDefGenerator) -> None:
             self.logger.debug(
                 "Applying generated scenario+controller changes to exp%s",
                 i)
-            expi = self.criteria.gen_exp_names(self.cmdopts)[i]
+            expi = self.criteria.gen_exp_names()[i]
             exp_output_root = self.batch_output_root / expi
             exp_input_root = self.batch_input_root / expi
 

sierra/core/generators/exp_generators.py

@@ -27,7 +27,7 @@
 # Project packages
 import sierra.core.generators.generator_factory as gf
 from sierra.core.experiment import spec, definition
-from sierra.core import types
+from sierra.core import types, batchroot
 import sierra.core.variables.batch_criteria as bc
 
 
@@ -41,20 +41,6 @@ class BatchExpDefGenerator:
         batch_config_template: Absolute path to the root template XML
                                configuration file.
 
-        batch_input_root: Root directory for all generated XML input files all
-                               experiments should be stored (relative to current
-                               dir or absolute). Each experiment will get a
-                               directory within this root to store the xml input
-                               files for the set of :term:`Experimental Runs
-                               <Experimental Run>` comprising an
-                               :term:`Experiment`; directory name determined by
-                               the batch criteria used.
-
-        batch_output_root: Root directory for all experiment outputs (relative
-                           to current dir or absolute). Each experiment will get
-                           a directory 'exp<n>' in this directory for its
-                           outputs.
-
         criteria: :class:`~sierra.core.variables.batch_criteria.BatchCriteria`
                   derived object instance created from cmdline definition.
 
@@ -66,6 +52,7 @@ class BatchExpDefGenerator:
 
     def __init__(self,
                  criteria: bc.IConcreteBatchCriteria,
+                 pathset: batchroot.PathSet,
                  controller_name: str,
                  scenario_basename: str,
                  cmdopts: types.Cmdopts) -> None:
@@ -77,8 +64,7 @@ def __init__(self,
         self.exp_template_stem = self.batch_config_template.stem
         self.batch_config_extension = None
 
-        self.batch_input_root = pathlib.Path(cmdopts['batch_input_root'])
-        self.batch_output_root = pathlib.Path(cmdopts['batch_output_root'])
+        self.pathset = pathset
 
         self.controller_name = controller_name
         self.scenario_basename = scenario_basename
@@ -118,7 +104,10 @@ def _create_exp_generator(self, exp_num: int):
             exp_num: Experiment number in the batch
         """
 
-        exp_spec = spec.ExperimentSpec(self.criteria, exp_num, self.cmdopts)
+        exp_spec = spec.ExperimentSpec(self.criteria,
+                                       self.pathset.input_root,
+                                       exp_num,
+                                       self.cmdopts)
         template_fpath = exp_spec.exp_input_root / self.exp_template_stem
         config_root = pathlib.Path(self.cmdopts['project_config_root'])
         scenario = gf.scenario_generator_create(controller=self.controller_name,

sierra/core/generators/generator_factory.py

@@ -82,15 +82,15 @@ def _do_tag_add(self,
             # the platform relies on added tags to calculate population sizes,
             # then this won't work.
             controllers = config.kYAML.controllers
-            assert hasattr(self.spec.criteria, 'n_robots'),\
+            assert hasattr(self.spec.criteria, 'n_agents'), \
                 (f"When using __UUID__ and tag_add in {controllers}, the batch "
                  "criteria must implement bc.IQueryableBatchCriteria")
-            n_robots = self.spec.criteria.n_robots(self.spec.exp_num)
+            n_agents = self.spec.criteria.n_agents(self.spec.exp_num)
 
-            assert n_robots > 0,\
+            assert n_agents > 0, \
                 "Batch criteria {self.spec.criteria} returned 0 robots?"
 
-            for robot_id in range(0, n_robots):
+            for robot_id in range(0, n_agents):
                 to_pp = copy.deepcopy(add)
                 pp_add = self._pp_for_tag_add(to_pp, robot_id)
                 exp_def.tag_add(pp_add.path,

sierra/core/models/interface.py

@@ -16,7 +16,7 @@
 
 # Project packages
 from sierra.core.variables import batch_criteria as bc
-from sierra.core import types
+from sierra.core import types, exproot, batchroot
 
 
 class IConcreteIntraExpModel1D(implements.Interface):
@@ -34,7 +34,8 @@ class IConcreteIntraExpModel1D(implements.Interface):
     def run(self,
             criteria: bc.IConcreteBatchCriteria,
             exp_num: int,
-            cmdopts: types.Cmdopts) -> tp.List[pd.DataFrame]:
+            cmdopts: types.Cmdopts,
+            pathset: exproot.PathSet) -> tp.List[pd.DataFrame]:
         """Run the model and generate a list of dataframes.
 
         Each dataframe can (potentially) target different graphs. All dataframes
@@ -97,7 +98,8 @@ class IConcreteIntraExpModel2D(implements.Interface):
     def run(self,
             criteria: bc.IConcreteBatchCriteria,
             exp_num: int,
-            cmdopts: types.Cmdopts) -> tp.List[pd.DataFrame]:
+            cmdopts: types.Cmdopts,
+            pathset: exproot.PathSet) -> tp.List[pd.DataFrame]:
         """Run the model and generate a list of dataframes.
 
         Each dataframe can (potentially) target a different graph. Each
@@ -150,7 +152,8 @@ class IConcreteInterExpModel1D(implements.Interface):
 
     def run(self,
             criteria: bc.IConcreteBatchCriteria,
-            cmdopts: types.Cmdopts) -> tp.List[pd.DataFrame]:
+            cmdopts: types.Cmdopts,
+            pathset: batchroot.PathSet) -> tp.List[pd.DataFrame]:
         """Run the model and generate list of dataframes.
 
         Each dataframe can (potentially) target a different graph. Each

sierra/core/pipeline/pipeline.py

@@ -33,7 +33,7 @@ class Pipeline:
     def __init__(self,
                  args: argparse.Namespace,
                  controller: tp.Optional[str],
-                 pathset: batchroot.PathSet) -> None:
+                 pathset: tp.Optional[batchroot.PathSet] = None) -> None:
         self.args = args
         self.logger = logging.getLogger(__name__)
 
@@ -145,11 +145,11 @@ def run(self) -> None:
                            self.batch_criteria).run()
 
         if 2 in self.args.pipeline:
-            PipelineStage2(self.cmdopts).run(self.batch_criteria)
+            PipelineStage2(self.cmdopts, self.pathset).run(self.batch_criteria)
 
         if 3 in self.args.pipeline:
             PipelineStage3(self.main_config,
-                           self.cmdopts).run(self.batch_criteria)
+                           self.cmdopts, self.pathset).run(self.batch_criteria)
 
         if 4 in self.args.pipeline:
             PipelineStage4(self.main_config,
@@ -158,7 +158,8 @@ def run(self) -> None:
         # not part of default pipeline
         if 5 in self.args.pipeline:
             PipelineStage5(self.main_config,
-                           self.cmdopts).run(self.args)
+                           self.cmdopts,
+                           self.pathset).run(self.args)
 
     def _load_config(self) -> None:
         self.logger.debug("Loading project config from '%s'",