diff --git a/docs/advanced.rst b/docs/advanced.rst index 94c68dd94..cb388b417 100644 --- a/docs/advanced.rst +++ b/docs/advanced.rst @@ -446,7 +446,7 @@ scheduler, but don't configure one. It's time to rectify that. # srun -N 50 -n 150 ./supermagic -a # Note that this would run in an sbatch script within an allocation # that conforms to the rest of the slurm settings. - - {sched.test_cmd} ./supermagic -a + - {sched.launch} ./supermagic -a - See ``pav show sched --vars `` for a listing of what variables are diff --git a/docs/results/parse.rst b/docs/results/parse.rst index 7ff3da77c..c4a9aaf71 100644 --- a/docs/results/parse.rst +++ b/docs/results/parse.rst @@ -329,7 +329,7 @@ depends on the **per\_file** attribute for the result parser. cmds: # Use the srun --output option to specify that results are # to be written to separate files. - - {{sched.test_cmd}} --output="%N.out" env + - {{sched.launch}} --output="%N.out" env result_parse: regex: diff --git a/docs/test_run_lifecycle.rst b/docs/test_run_lifecycle.rst index 62ce56693..e14696705 100644 --- a/docs/test_run_lifecycle.rst +++ b/docs/test_run_lifecycle.rst @@ -217,7 +217,7 @@ something like this: "result": "last_login > {{pav.timestamp - time_limit}}"}, "regex": [{"key": "last_login", "regex": "Last Login: (\\d+)"}]}, - "run": {"cmds": ["{{sched.test_cmd}} ./test1 $ARGS"], + "run": {"cmds": ["{{sched.launch}} ./test1 $ARGS"], "env": {"ARGS": "-u {{user}}"}, "modules": ["gcc"]}, "variables": {"hours": 5, @@ -457,4 +457,4 @@ Mark the Test as Complete Finally, the test is marked as complete by saving a 'RUN_COMPLETE' file in the test's run directory. Pavilion uses this to quickly determine which tests -might still be running. \ No newline at end of file +might still be running. diff --git a/docs/tests/scheduling.rst b/docs/tests/scheduling.rst index d78448c5e..5e19cec34 100644 --- a/docs/tests/scheduling.rst +++ b/docs/tests/scheduling.rst @@ -111,9 +111,9 @@ command. | | | 1. tasks_total | True | 180 | Total tasks to create, based on number of nodes | | | actually acquired. - test_cmd | True | srun -N 5 -w no | Construct a cmd to run a process under this + launch | True | srun -N 5 -w no | Construct a cmd to run a process under this | | de[05-10],node2 | scheduler, with the criteria specified by this test. - | | 3 -n 20 | + | | 3 -n 20 | (used to be `sched.test_cmd`) .. _tests.scheduling.jobs: @@ -342,9 +342,9 @@ wrapper command before actually running the intended command. cmds: # The run command will be `srun -N1 -p standard valgrind ./supermagic -a` # It will run `valgrind ./supermagic -a` on the allocation - - '{{sched.test_cmd}} ./supermagic -a' + - '{{sched.launch}} ./supermagic -a' -When using the ``raw`` scheduler, the ``{{sched.test_cmd}}`` normally returns an empty string. You can +When using the ``raw`` scheduler, the ``{{sched.launch}}`` normally returns an empty string. You can use the wrapper setting to control a different scheduler directly. .. code-block:: yaml @@ -359,5 +359,5 @@ use the wrapper setting to control a different scheduler directly. run: cmds: # With the schedule wrapper, this will be `mpirun -np 2 ./supermagic -a` - - '{{sched.test_cmd}} ./supermagic -a' + - '{{sched.launch}} ./supermagic -a' diff --git a/docs/tests/values.rst b/docs/tests/values.rst index dfb9636e6..ed31dae44 100644 --- a/docs/tests/values.rst +++ b/docs/tests/values.rst @@ -46,7 +46,7 @@ not Python3 code.** cmds: # In the simple case, expressions are often just a variable # reference. - - "{{sched.test_cmds}} ./my_cmd" + - "{{sched.launch}} ./my_cmd" Types of Expressions ^^^^^^^^^^^^^^^^^^^^ diff --git a/examples/acceptance/tests/ior.yaml b/examples/acceptance/tests/ior.yaml index dfa526262..d10eed96a 100644 --- a/examples/acceptance/tests/ior.yaml +++ b/examples/acceptance/tests/ior.yaml @@ -273,11 +273,11 @@ scaling_study: - '[ -d {{scratch.path}}/ior-test-{{dirname}} ] && rm -Rf {{scratch.path}}/ior-test-{{dirname}}' - 'mkdir {{scratch.path}}/ior-test-{{dirname}} && pushd {{scratch.path}}/ior-test-{{dirname}}' - '# this writes out the files' - - '{{sched.test_cmd}} $IOR_EXE "${test_variables} -w' + - '{{sched.launch}} $IOR_EXE "${test_variables} -w' - 'if [[ {{numnodes}} -ne 1 ]]' - 'then' - ' # this reads those files' - - ' {{sched.test_cmd}} $IOR_EXE -E -C -Q {{taskspernode}} -r' + - ' {{sched.launch}} $IOR_EXE -E -C -Q {{taskspernode}} -r' - 'fi' - 'cd ..' - 'rm -rf {{scratch.path}}/ior-test-{{dirname}}' diff --git a/examples/acceptance/tests/pynamic.yaml b/examples/acceptance/tests/pynamic.yaml index 36d8d45b6..031100fee 100644 --- a/examples/acceptance/tests/pynamic.yaml +++ b/examples/acceptance/tests/pynamic.yaml @@ -110,7 +110,7 @@ base: # Run commands. cmds: - 'cd pynamic-pyMPI-2.6a1' - - '{{sched.test_cmd}} ./pynamic-mpi4py $(date +%s)' + - '{{sched.launch}} ./pynamic-mpi4py $(date +%s)' result_parse: regex: diff --git a/examples/acceptance/tests/umt.yaml b/examples/acceptance/tests/umt.yaml index 2defb36b2..a3d501f2d 100644 --- a/examples/acceptance/tests/umt.yaml +++ b/examples/acceptance/tests/umt.yaml @@ -208,7 +208,7 @@ _base: # seconds to update all the information. Sleeping for a few seconds allows for the seff command # to report the data accurately. cmds: - - "{{sched.test_cmd}} ./umt2015-crossroads/Teton/SuOlsonTest {{umt_run.gridfilename}} + - "{{sched.launch}} ./umt2015-crossroads/Teton/SuOlsonTest {{umt_run.gridfilename}} {{umt_run.groups}} {{umt_run.quadtype}} {{umt_run.order}} {{umt_run.polar}} {{umt_run.azim}} ; job_id=${SLURM_JOB_ID}" - "sleep 10s" diff --git a/examples/acceptance/tests/vpic.yaml b/examples/acceptance/tests/vpic.yaml index 23a86e5c6..3e2a930ed 100644 --- a/examples/acceptance/tests/vpic.yaml +++ b/examples/acceptance/tests/vpic.yaml @@ -265,7 +265,7 @@ _base: cmds: - ./build/bin/vpic {{vpic_input_deck}} - 'top -b -u {{user}} > top_out.txt 2>&1 &' - - '{{vpic_run.time_cmd}} {{sched.test_cmd}} {{vpic_run.numa}} ./lpi-input.Linux --tpp 1' + - '{{vpic_run.time_cmd}} {{sched.launch}} {{vpic_run.numa}} ./lpi-input.Linux --tpp 1' schedule: nodes: '{{nnodes}}' diff --git a/examples/pre-packaged/tests/hello_mpi.yaml b/examples/pre-packaged/tests/hello_mpi.yaml index 9391f307d..60b41223f 100644 --- a/examples/pre-packaged/tests/hello_mpi.yaml +++ b/examples/pre-packaged/tests/hello_mpi.yaml @@ -44,4 +44,4 @@ base: - "{{mpis}}" cmds: - "set -x" - - "{{sched.test_cmd}} ./mpi_hello" + - "{{sched.launch}} ./mpi_hello" diff --git a/examples/pre-packaged/tests/hpcg.yaml b/examples/pre-packaged/tests/hpcg.yaml index 6cc84d1f3..7e7d4c779 100644 --- a/examples/pre-packaged/tests/hpcg.yaml +++ b/examples/pre-packaged/tests/hpcg.yaml @@ -25,7 +25,7 @@ base: run: modules: ["{{compilers}}", "{{mpis}}"] - cmds: "{{sched.test_cmd}} ./bin/xhpcg" + cmds: "{{sched.launch}} ./bin/xhpcg" result_parse: regex: diff --git a/examples/pre-packaged/tests/imb.yaml b/examples/pre-packaged/tests/imb.yaml index efcf0af2b..7356425c6 100644 --- a/examples/pre-packaged/tests/imb.yaml +++ b/examples/pre-packaged/tests/imb.yaml @@ -44,4 +44,4 @@ base: cmds: - cd src_c - - "{{sched.test_cmd}} ./IMB-MPI1 -npmin {{sched.alloc_cpu_total}} -mem 50 alltoall" + - "{{sched.launch}} ./IMB-MPI1 -npmin {{sched.alloc_cpu_total}} -mem 50 alltoall" diff --git a/examples/pre-packaged/tests/ior.yaml b/examples/pre-packaged/tests/ior.yaml index 670fcb01a..51a293858 100644 --- a/examples/pre-packaged/tests/ior.yaml +++ b/examples/pre-packaged/tests/ior.yaml @@ -76,7 +76,7 @@ base: - " rm -rf ior-test-{{block_size}}-{{a_flag}}-{{scratch.name}}" - " mkdir ior-test-{{block_size}}-{{a_flag}}-{{scratch.name}}" - " cd ior-test-{{block_size}}-{{a_flag}}-{{scratch.name}}" - - " {{sched.test_cmd}} {{srun_opts}} $bloc/src/ior -a {{a_flag}} -b {{block_size}} -c -e -F -i 2" + - " {{sched.launch}} {{srun_opts}} $bloc/src/ior -a {{a_flag}} -b {{block_size}} -c -e -F -i 2" - " cd .." - " rm -rf {{scratch.path}}/ior-test-{{block_size}}-{{a_flag}}-{{scratch.name}}" - "fi" @@ -180,7 +180,7 @@ xroads_load1: - "rm -rf ior-test-{{outname}}" - "mkdir ior-test-{{outname}}" - "cd ior-test-{{outname}}" - - "{{sched.test_cmd}} {{srun_opts}} $bloc/src/ior -f $bloc/inp" + - "{{sched.launch}} {{srun_opts}} $bloc/src/ior -f $bloc/inp" - "cd .." - "rm -rf {{scratch.path}}/ior-test-{{outname}}" diff --git a/examples/pre-packaged/tests/stream.yaml b/examples/pre-packaged/tests/stream.yaml index ba94013c1..e83899783 100644 --- a/examples/pre-packaged/tests/stream.yaml +++ b/examples/pre-packaged/tests/stream.yaml @@ -18,4 +18,4 @@ base: - "{{compilers}}" - "{{mpis}}" cmds: - - "{{sched.test_cmd}} ./stream" + - "{{sched.launch}} ./stream" diff --git a/examples/pre-packaged/tests/supermagic.yaml b/examples/pre-packaged/tests/supermagic.yaml index 7827aeb62..b6bac3f24 100644 --- a/examples/pre-packaged/tests/supermagic.yaml +++ b/examples/pre-packaged/tests/supermagic.yaml @@ -41,7 +41,7 @@ base: - "{{mpis}}" cmds: - - "{{sched.test_cmd}} ./supermagic" + - "{{sched.launch}} ./supermagic" result_parse: regex: diff --git a/lib/pavilion/schedulers/plugins/flux.py b/lib/pavilion/schedulers/plugins/flux.py index c8ed48505..27b606324 100644 --- a/lib/pavilion/schedulers/plugins/flux.py +++ b/lib/pavilion/schedulers/plugins/flux.py @@ -66,10 +66,10 @@ class FluxVars(SchedulerVariables): EXAMPLE = SchedulerVariables.EXAMPLE.copy() EXAMPLE.update({ - 'test_cmd': 'flux run -x -N 5 -n 20', + 'launch': 'flux run -x -N 5 -n 20', }) - def _test_cmd(self): + def _launch(self): """Construct a cmd to run a process under this scheduler, with the criteria specified by this test. """ @@ -91,13 +91,13 @@ def _test_cmd(self): return ' '.join(cmd) @dfr_var_method - def test_cmd(self): + def launch(self): """Calls the actual test command and then wraps the return with the wrapper provided in the schedule section of the configuration.""" # Removes all the None values to avoid getting a TypeError while trying to # join two commands - return ' '.join(filter(lambda item: item is not None, [self._test_cmd(), + return ' '.join(filter(lambda item: item is not None, [self._launch(), self._sched_config['wrapper']])) @@ -210,11 +210,11 @@ def _get_config_elems(self): yc.ListElem(name='fluxrun_extra', sub_elem=yc.StrElem(), help_text="Extra arguments to pass to flux run as part of " - "the 'sched.test_cmd' variable."), + "the 'sched.launch' variable."), yc.ListElem(name='fluxbatch_extra', sub_elem=yc.StrElem(), help_text="Extra arguments to pass to flux batch as part of " - "the 'sched.test_cmd' variable."), + "the 'sched.launch' variable."), ] defaults = { diff --git a/lib/pavilion/schedulers/plugins/slurm.py b/lib/pavilion/schedulers/plugins/slurm.py index 438175504..077186bd7 100644 --- a/lib/pavilion/schedulers/plugins/slurm.py +++ b/lib/pavilion/schedulers/plugins/slurm.py @@ -133,10 +133,10 @@ class SlurmVars(SchedulerVariables): EXAMPLE = SchedulerVariables.EXAMPLE.copy() EXAMPLE.update({ - 'test_cmd': 'srun -N 5 -w node[05-10],node23 -n 20', + 'launch': 'srun -N 5 -w node[05-10],node23 -n 20', }) - def _test_cmd(self): + def _launch(self): """Construct a cmd to run a process under this scheduler, with the criteria specified by this test. """ @@ -171,13 +171,13 @@ def _test_cmd(self): return ' '.join(cmd) @dfr_var_method - def test_cmd(self): + def launch(self): """Calls the actual test command and then wraps the return with the wrapper provided in the schedule section of the configuration.""" # Removes all the None values to avoid getting a TypeError while trying to # join two commands - return ' '.join(filter(lambda item: item is not None, [self._test_cmd(), + return ' '.join(filter(lambda item: item is not None, [self._launch(), self._sched_config['wrapper']])) def slurm_float(val): @@ -276,7 +276,7 @@ def _get_config_elems(self): yc.ListElem(name='srun_extra', sub_elem=yc.StrElem(), help_text="Extra arguments to pass to srun as part of the " - "'sched.test_cmd' variable."), + "'sched.launch' variable."), yc.ListElem(name='sbatch_extra', sub_elem=yc.StrElem(), help_text="Extra arguments to add as sbatch header lines. " diff --git a/lib/pavilion/schedulers/vars.py b/lib/pavilion/schedulers/vars.py index e7a408aab..496efb85c 100644 --- a/lib/pavilion/schedulers/vars.py +++ b/lib/pavilion/schedulers/vars.py @@ -23,7 +23,7 @@ class of this that contains all the variable functions it provides. This class is meant to be inherited from - each scheduler can provide its own set of variables in addition to these defaults, and may also provide different implementations of -each variable. Most schedulers can get away with overriding one variable - the 'test_cmd' +each variable. Most schedulers can get away with overriding one variable - the 'launch' method. See the documentation for that method below for more information. Return values of all variables should be the same format as those allowed by regular test @@ -47,6 +47,7 @@ class of this that contains all the variable functions it provides. 'test_min_cpus': '4', 'test_min_mem': '32', 'tasks_total': '180', + 'test_cmd': '# This is an alias to "launch"', } # Scheduler variable errors are deferred. We'll handle them later we we create @@ -155,7 +156,7 @@ def partition(self): return self._sched_config['partition'] or '' - def _test_cmd(self): + def _launch(self): """The command to prepend to a line to kick it off under the scheduler. This should return the command needed to start one or more MPI processes within @@ -176,14 +177,20 @@ def _test_cmd(self): return '' - @var_method + @dfr_var_method def test_cmd(self): + """Alias to the launch command""" + + return self.launch() + + @var_method + def launch(self): """Calls the actual test command and then wraps the result with the wrapper provided in the schedule section of the configuration.""" # Removes all the None values to avoid getting a TypeError while trying to # join two commands - return ''.join(filter(lambda item: item is not None, [self._test_cmd(), + return ''.join(filter(lambda item: item is not None, [self._launch(), self._sched_config['wrapper']])) @dfr_var_method diff --git a/test/data/pav_config_dir/tests/sched_errors.yaml b/test/data/pav_config_dir/tests/sched_errors.yaml index 7436f0e74..fd0ddb4a5 100644 --- a/test/data/pav_config_dir/tests/sched_errors.yaml +++ b/test/data/pav_config_dir/tests/sched_errors.yaml @@ -4,7 +4,7 @@ a_error: scheduler: dummy run: cmds: - - '{{sched.test_cmd}} echo "foo"' + - '{{sched.launch}} echo "foo"' b_skipped: inherits_from: a_error @@ -25,4 +25,4 @@ d_no_nodes: schedule: nodes: 5 - reservation: 'no_exist' \ No newline at end of file + reservation: 'no_exist' diff --git a/test/tests/sched_tests.py b/test/tests/sched_tests.py index 189ed5d9c..fbc7c58bf 100644 --- a/test/tests/sched_tests.py +++ b/test/tests/sched_tests.py @@ -74,7 +74,7 @@ def test_sched_var_values(self): NodeSet(frozenset(['node05', 'node06', 'node07']))] expected = { - 'test_cmd': '', + 'launch': '', 'tasks_per_node': '1', 'chunk_ids': ['0', '1'], 'chunk_size': '3', @@ -128,7 +128,7 @@ def test_sched_var_values_basic(self): chunks = None expected = { - 'test_cmd': '', + 'launch': '', 'tasks_per_node': '1', 'chunk_ids': [], 'chunk_size': '', @@ -598,7 +598,7 @@ def test_wrapper(self): # The wrapper can by anything a command or even a string test_cfg['schedule'] = {'wrapper': 'echo'} - test_cfg['run']['cmds'] = ['{{sched.test_cmd}} "this is the wrapper test"'] + test_cfg['run']['cmds'] = ['{{sched.launch}} "this is the wrapper test"'] test = self._quick_test(test_cfg, finalize=False) diff --git a/test/tests/slurm_tests.py b/test/tests/slurm_tests.py index fcfb7fea7..736470ad2 100644 --- a/test/tests/slurm_tests.py +++ b/test/tests/slurm_tests.py @@ -297,7 +297,7 @@ def test_cancel(self): slurm = pavilion.schedulers.get_plugin('slurm') cfg = self._quick_test_cfg() - cfg['run']['cmds'] = ['{{sched.test_cmd}} sleep 10'] + cfg['run']['cmds'] = ['{{sched.launch}} sleep 10'] cfg.update(self.slurm_mode) cfg['schedule']['nodes'] = '5' cfg['scheduler'] = 'slurm' @@ -320,7 +320,7 @@ def test_slurm_params(self): slurm = pavilion.schedulers.get_plugin('slurm') cfg = self._quick_test_cfg() cfg.update(self.slurm_mode) - cfg['run']['cmds'] = ['{{sched.test_cmd}} hostname'] + cfg['run']['cmds'] = ['{{sched.launch}} hostname'] cfg['schedule']['nodes'] = '5' cfg['schedule']['slurm'] = { 'sbatch_extra': ['--comment "Hiya!"'], @@ -344,7 +344,7 @@ def test_slurm_kickoff_shared(self): slurm = pavilion.schedulers.get_plugin('slurm') cfg = self._quick_test_cfg() cfg.update(self.slurm_mode) - cfg['run']['cmds'] = ['{{sched.test_cmd}} hostname'] + cfg['run']['cmds'] = ['{{sched.launch}} hostname'] cfg['schedule']['nodes'] = '3' cfg['schedule']['node_state'] = 'available' cfg['schedule']['share_allocation'] = 'max' @@ -380,7 +380,7 @@ def test_slurm_kickoff_indi(self): slurm = pavilion.schedulers.get_plugin('slurm') cfg = self._quick_test_cfg() cfg.update(self.slurm_mode) - cfg['run']['cmds'] = ['{{sched.test_cmd}} hostname'] + cfg['run']['cmds'] = ['{{sched.launch}} hostname'] cfg['schedule']['nodes'] = 'all' cfg['schedule']['chunking'] = {'size': '3'} cfg['schedule']['share_allocation'] = 'False' @@ -412,7 +412,7 @@ def test_slurm_kickoff_flex(self): slurm = pavilion.schedulers.get_plugin('slurm') cfg = self._quick_test_cfg() cfg.update(self.slurm_mode) - cfg['run']['cmds'] = ['{{sched.test_cmd}} hostname'] + cfg['run']['cmds'] = ['{{sched.launch}} hostname'] cfg['schedule']['nodes'] = '3' cfg['schedule']['share_allocation'] = 'False' cfg['chunk'] = '0' @@ -447,7 +447,7 @@ def test_mpirun(self): slurm = pavilion.schedulers.get_plugin('slurm') cfg = self._quick_test_cfg() cfg.update(self.slurm_mode) - cfg['run']['cmds'] = ['{{sched.test_cmd}} hostname'] + cfg['run']['cmds'] = ['{{sched.launch}} hostname'] cfg['run']['modules'] = ['gcc', 'openmpi'] cfg['schedule']['nodes'] = '5' cfg['schedule']['slurm'] = { @@ -470,7 +470,7 @@ def test_mpirun(self): slurm = pavilion.schedulers.get_plugin('slurm') cfg = self._quick_test_cfg() cfg.update(self.slurm_mode) - cfg['run']['cmds'] = ['{{sched.test_cmd}} hostname'] + cfg['run']['cmds'] = ['{{sched.launch}} hostname'] cfg['run']['modules'] = ['gcc', 'openmpi'] cfg['schedule']['nodes'] = '1' cfg['schedule']['slurm'] = {'mpi_cmd': 'mpirun'}