Merge branch 'develop' into 745-build-opts

.github/workflows/demo.yml

@@ -2,9 +2,9 @@ name: demo_configs
 
 on:
   push:
-    branches: [ master ]
+    branches: [ develop  ]
   pull_request:
-    branches: [ master ]
+    branches: [ develop ]
 
 jobs:
   demo_runs:
@@ -13,18 +13,18 @@ jobs:
 
     steps:
     - uses: actions/checkout@v4
-      
+
     - name: Set up Python 3.10
       uses: actions/setup-python@v2
       with:
         python-version: "3.10"
-    
+
     - name: install-deps
       run: sudo apt install gcc
 
-    - name: 
+    - name:
       run: |
         source examples/demo/activate.sh
         pav run -v QUIET demo advanced buildit
-        pav wait 
+        pav wait
         pav results --all-passed

.github/workflows/unittests.yml

@@ -2,9 +2,9 @@ name: unitests
 
 on:
   push:
-    branches: [ master ]
+    branches: [ develop ]
   pull_request:
-    branches: [ master ]
+    branches: [ develop ]
 
 jobs:
   # The debug job is meant for debugging CI related issues.
@@ -18,36 +18,36 @@ jobs:
     - uses: actions/checkout@v2
       with:
         submodules: recursive
-      
+
     - name: Set up Python 3.10
       uses: actions/setup-python@v2
       with:
         python-version: "3.10"
-    
+
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
         pip install -r requirements.txt
         pip install -r test/requirements.txt
         pip install -r docs/requirements.txt
         pip install coverage
-        
+
     - name: setup
       run: |
         ln -s pavilion.yaml.ci test/data/pav_config_dir/pavilion.yaml
         echo "working_dir: $(pwd)/test/working_dir" >> test/data/pav_config_dir/pavilion.yaml
         #echo "spack_path: $(pwd)/test/spack" >> test/data/pav_config_dir/pavilion.yaml
         #./test/utils/spack_setup test
-    
+
     # Set run_tests to run only the specific tests you need to fix.
     - name: run_tests
-      run: | 
+      run: |
         ./test/run_tests
 
     - name: zip_working_dir
       if: always()
       run: |
-        # If we don't zip this ourselves, it will run infinitely due to 
+        # If we don't zip this ourselves, it will run infinitely due to
         # symlink loops.
         cd test
         ./utils/make_symlinks_relative working_dir
@@ -66,12 +66,12 @@ jobs:
 
     steps:
     - uses: actions/checkout@v2
-      
+
     - name: Set up Python 3.10
       uses: actions/setup-python@v2
       with:
         python-version: "3.10"
-    
+
     - name: setup_via_venv
       run: |
         mv .git .not-git
@@ -90,12 +90,12 @@ jobs:
     - uses: actions/checkout@v2
       with:
         submodules: recursive
-      
+
     - name: Set up Python 3.10
       uses: actions/setup-python@v2
       with:
         python-version: "3.10"
-    
+
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
@@ -108,9 +108,9 @@ jobs:
         ln -s pavilion.yaml.ci test/data/pav_config_dir/pavilion.yaml
         echo "working_dir: $(pwd)/test/working_dir" >> test/data/pav_config_dir/pavilion.yaml
         echo "spack_path: $(pwd)/test/spack" >> test/data/pav_config_dir/pavilion.yaml
-         
+
     - name: run_tests
-      run: | 
+      run: |
         sudo -E su $USER -c "PATH=$PATH; ./test/run_tests -q -o style -o debug_prints"
 
   docs:
@@ -120,12 +120,12 @@ jobs:
     - uses: actions/checkout@v2
       with:
         submodules: recursive
-      
+
     - name: Set up Python 3.10
       uses: actions/setup-python@v2
       with:
         python-version: "3.10"
-    
+
     - name: Install dependencies
       run: |
         pip install -r requirements.txt
@@ -137,26 +137,26 @@ jobs:
         ln -s pavilion.yaml.ci test/data/pav_config_dir/pavilion.yaml
         echo "working_dir: $(pwd)/test/working_dir" >> test/data/pav_config_dir/pavilion.yaml
         echo "spack_path: $(pwd)/test/spack" >> test/data/pav_config_dir/pavilion.yaml
-         
+
     - name: run_tests
       run: ./test/run_tests -o 'doc*' -q
 
   unittests-3_6:
     runs-on: ubuntu-20.04
-    needs: 
+    needs:
         - style
         - docs
 
     steps:
     - uses: actions/checkout@v2
       with:
         submodules: recursive
-      
+
     - name: Set up Python 3.6
       uses: actions/setup-python@v2
       with:
         python-version: 3.6
-    
+
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
@@ -167,23 +167,23 @@ jobs:
         sphinx-build --version
         pylint --version
 
-        
+
     - name: setup
       run: |
         ln -s pavilion.yaml.ci test/data/pav_config_dir/pavilion.yaml
         echo "working_dir: $(pwd)/test/working_dir" >> test/data/pav_config_dir/pavilion.yaml
         echo "spack_path: $(pwd)/test/spack" >> test/data/pav_config_dir/pavilion.yaml
         ./test/utils/spack_setup test
-        
+
     - name: run_tests
-      run: | 
-        ./test/run_tests 
+      run: |
+        ./test/run_tests
         #coverage xml
 
     - name: zip_working_dir
       if: failure()
       run: |
-        # If we don't zip this ourselves, it will run infinitely due to 
+        # If we don't zip this ourselves, it will run infinitely due to
         # symlink loops.
         cd test
         ./utils/make_symlinks_relative working_dir
@@ -201,45 +201,45 @@ jobs:
 #      with:
 #        fail_ci_if_error: true
 #        token: c1100f28-f36f-4b92-8ba5-ec907a1bfea6
- 
+
   unittests-3_10:
     runs-on: ubuntu-latest
-    needs: 
+    needs:
         - style
         - docs
 
     steps:
     - uses: actions/checkout@v2
       with:
         submodules: recursive
-      
+
     - name: Set up Python 3.10
       uses: actions/setup-python@v2
       with:
         python-version: "3.10"
-    
+
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
         pip install -r requirements.txt
         pip install -r test/requirements.txt
         pip install -r docs/requirements.txt
-    
+
     - name: setup
       run: |
         ln -s pavilion.yaml.ci test/data/pav_config_dir/pavilion.yaml
         echo "working_dir: $(pwd)/test/working_dir" >> test/data/pav_config_dir/pavilion.yaml
         echo "spack_path: $(pwd)/test/spack" >> test/data/pav_config_dir/pavilion.yaml
         ./test/utils/spack_setup test
-        
+
     - name: run_tests
-      run: | 
+      run: |
         sudo -E su $USER -c "PATH=$PATH; ./test/run_tests"
 
     - name: zip_working_dir
       if: failure()
       run: |
-        # If we don't zip this ourselves, it will run infinitely due to 
+        # If we don't zip this ourselves, it will run infinitely due to
         # symlink loops.
         cd test
         ./utils/make_symlinks_relative working_dir

docs/tests/run.rst

@@ -112,6 +112,25 @@ including to such large numbers that your test will never time out.
 
 .. _tests.run.extending_commands:
 
+concurrent
+^^^^^^^^^^
+
+When tests are running in the same allocation (under the same batch script), you can specify
+that more than one test can run concurrently with others. By default, tests scheduled through
+cluster schedulers (slurm/flux) have this set to 1 - forcing tests to run serially. The raw scheduler, in
+contrast, sets it to more (see ``pav show sched raw --config``).
+
+For more about shared allocations, see  :ref:`tests.scheduling.job_sharing`.
+
+.. code-block:: yaml
+
+   base:
+     run:
+       # This test is ok with running alongside up to four other tests in an allocation.
+       concurrent: 5
+       cmds:
+         - ..
+
 Extending Commands
 ~~~~~~~~~~~~~~~~~~
 
@@ -148,4 +167,4 @@ For the 'next' test, we would end up with a list of run commands like this:
     - b
     - c
     - 'append_a'
-    - 'append_b'
+    - 'append_b'

docs/tests/scheduling.rst

@@ -227,13 +227,17 @@ Job Sharing
 -----------
 
 On an advanced scheduler, when two tests have the same job parameters, they are automatically
-scheduled together in the same job allocation. The kickoff script for that job will start the
-tests serially, and the result of each test run does not effect the others.
+scheduled together in the same job allocation. The kickoff script for that job will run the tests
+concurrently up to the limit set by ``run.concurrent`` for each test.
 
 Job sharing makes the most sense for short tests that cover a wide range of nodes - such tests
-often take longer to set up the allocation than they do to run.
+often take longer to set up the allocation than they do to run. By default (when
+``schedule.share_allocation`` is set to ``true``), Pavilion will try to balance the sharing with
+the number of nodes available - effectively distributing the test runs into jobs that span the
+nodes.
 
-This is enabled by default. It can be disabled through the ``schedule.share_allocation`` option.
+With ``schedule.share_allocation`` set to ``max``, Pavilion forces as many test runs into the same
+job as possible.
 
 .. _tests.scheduling.chunking:
 
@@ -340,7 +344,7 @@ wrapper command before actually running the intended command.
                 # It will run `valgrind ./supermagic -a` on the allocation
                 - '{{sched.test_cmd}} ./supermagic -a'
 
-When using the ``raw`` scheduler, the ``{{sched.test_cmd}}`` normally returns an empty string. You can 
+When using the ``raw`` scheduler, the ``{{sched.test_cmd}}`` normally returns an empty string. You can
 use the wrapper setting to control a different scheduler directly.
 
 .. code-block:: yaml

docs/tests/values.rst

@@ -70,7 +70,7 @@ identically to Python3 (with one noted exception). This includes:
 - Concatenation ``"hello " .. "world"`` and ``[1, 2 ,3] .. [4, 5, 6]``
 
 List Operations
-```````````````
+^^^^^^^^^^^^^^^
 
 When using math operations with list values, the operation is applied
 recursively to each element. Operations between two lists require that the
@@ -91,6 +91,21 @@ of items.
             # This would produce (1*4 + 2*4 + 3*2 + 4*1.5)/4 == 6.0
             mult_avg: '{{avg(nums*mult)}}'
 
+List Indexing and Slicing
+^^^^^^^^^^^^^^^^^^^^^^^^^
+
+You can reference individual items and slices from lists both from variables when they're returned
+from functions. The syntax ``val[start:end]`` mirrors that of Python.  You can also use the
+Pavilion classic syntax of ``val.1`` or ``val.*`` to access individual items or get the full list of
+items (``val`` is the same as ``val.0`` or ``val[0]``).
+
+.. code-block::
+
+    list_expr_test:
+        variables:
+            nums: '{{ range(1, 10)[4] }}'
+            vals: [4, 4, 2, 1.5]
+            some_sum: '{{ sum(vals[1:3]) }}'
 
 Types and End Results
 ^^^^^^^^^^^^^^^^^^^^^