Uenv ci venv activation

repos/c2sm/packages/icon/package.py

@@ -581,6 +581,7 @@ def configure_args(self):
         # Check for DSL variants and set corresponding Liskov options
         dsl = self.spec.variants['dsl'].value
         if dsl != ('none', ):
+            args.append('--disable-venv-activation')
             if 'substitute' in dsl:
                 args.append('--enable-liskov=substitute')
             elif 'verify' in dsl:

setup-env.sh

@@ -8,9 +8,17 @@ else
     machine="$( "$parent_dir"/src/machine.sh )"
 fi
 
-export SPACK_SYSTEM_CONFIG_PATH="$parent_dir"/sysconfigs/"$machine"
-export SPACK_USER_CONFIG_PATH="$parent_dir"/user-config
-export SPACK_USER_CACHE_PATH="$parent_dir"/user-cache
+if [[ ${machine:0:1} == "/" ]]; then
+    export SPACK_SYSTEM_CONFIG_PATH="$machine"/config
+    export SPACK_USER_CONFIG_PATH="$parent_dir"/sysconfigs/uenv
+    export SPACK_USER_CACHE_PATH="$parent_dir"/user-cache/uenv
+else
+    export SPACK_SYSTEM_CONFIG_PATH="$parent_dir"/sysconfigs/"$machine"
+    export SPACK_USER_CONFIG_PATH="$parent_dir"/user-config
+    export SPACK_USER_CACHE_PATH="$parent_dir"/user-cache/"$machine"
+fi
+
+
 . "$parent_dir"/spack/share/spack/setup-env.sh
 
 echo Spack configured for "$machine".

src/spack_commands.py

@@ -16,7 +16,8 @@ def log_with_spack(command: str,
                    log_filename: str = None,
                    cwd=None,
                    env=None,
-                   srun=False) -> None:
+                   allow_srun=False,
+                   uenv=None) -> None:
     """
     Executes the given command while spack is loaded and writes the output into the log file.
     If log_filename is None, command is used to create one.
@@ -27,13 +28,27 @@ def log_with_spack(command: str,
 
     # Setup spack env
     spack_env = f'. {spack_c2sm_path}/setup-env.sh'
+    if uenv:
+        spack_env += ' /user-environment'
+
+    # Only use srun as Jenkins user and on machines where it is available
+    use_srun = allow_srun and getpass.getuser() == 'jenkins' and machine_name(
+    ) in ['daint', 'tsa']
+
+    uenv_args = ''
+    if uenv:
+        uenv_mount_point = f'{uenv}:/user-environment'
+        if use_srun:
+            uenv_args = '--uenv=' + uenv_mount_point
+        else:
+            uenv_args = 'squashfs-mount ' + uenv_mount_point + ' -- '
 
     # Distribute work with 'srun'
-    if srun and getpass.getuser() == 'jenkins':
+    if use_srun:
         # The '-c' argument should be in sync with
         # sysconfig/<machine>/config.yaml config:build_jobs for max efficiency
+
         srun = {
-            'balfrin': '',
             'daint': 'srun -t 02:00:00 -C gpu -A g110 -c 12 -n 1',
             'tsa': 'srun -t 02:00:00 -c 6',
         }[machine_name()]
@@ -47,6 +62,9 @@ def log_with_spack(command: str,
     with log_file.open('a') as f:
         f.write(machine_name())
         f.write('\n')
+        if uenv:
+            f.write(f'uenv: {uenv}')
+            f.write('\n')
         f.write(command)
         f.write('\n\n')
 
@@ -55,7 +73,7 @@ def log_with_spack(command: str,
     # The output is streamed as directly as possible to the log_file to avoid buffering and potentially losing buffered content.
     # '2>&1' redirects stderr to stdout.
     ret = subprocess.run(
-        f'{spack_env}; {env_activate} ({srun} {command}) >> {log_file} 2>&1',
+        f'({srun} {uenv_args} bash -c "{spack_env}; {env_activate} {command}") >> {log_file} 2>&1',
         cwd=cwd,
         check=False,
         shell=True)

sysconfigs/uenv/concretizer.yaml

@@ -0,0 +1,2 @@
+concretizer:
+  reuse: true

sysconfigs/uenv/config.yaml

@@ -0,0 +1,6 @@
+config:
+  build_stage: '$spack/../spack-build_stage'
+  test_stage: '$spack/../spack-test_stage'
+  misc_cache: '$spack/../spack-misc_cache'
+  build_jobs: 12
+  db_lock_timeout: 120

sysconfigs/uenv/repos.yaml

@@ -0,0 +1,2 @@
+repos:
+  - '$spack/../repos/c2sm'

test/system_test.py

@@ -1,11 +1,9 @@
-import unittest
 import pytest
 import subprocess
 import sys
 import os
 import uuid
 import shutil
-from pathlib import Path
 import inspect
 
 spack_c2sm_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),
@@ -14,6 +12,8 @@
 sys.path.append(os.path.normpath(spack_c2sm_path))
 from src import machine_name, log_with_spack, sanitized_filename
 
+PREPOST = '/scratch/mch/leclairm/uenvs/images/pre-post_v0.sqfs'
+
 
 @pytest.fixture(scope='function')
 def devirt_env():
@@ -39,7 +39,7 @@ def compose_logfilename(spec):
     return sanitized_filename(func_name + '-' + spec)
 
 
-def spack_install(spec: str, test_root: bool = True):
+def spack_install(spec: str, test_root: bool = True, uenv: str = None):
     """
     Tests 'spack install' of the given spec and writes the output into the log file.
     """
@@ -50,13 +50,15 @@ def spack_install(spec: str, test_root: bool = True):
     log_with_spack(f'spack spec {spec}',
                    'system_test',
                    log_filename,
-                   srun=False)
+                   allow_srun=False,
+                   uenv=uenv)
 
     test_arg = "--test=root" if test_root else ""
     log_with_spack(f'spack install -n -v {test_arg} {spec}',
                    'system_test',
                    log_filename,
-                   srun=not spec.startswith('icon '))
+                   allow_srun=not spec.startswith('icon '),
+                   uenv=uenv)
 
 
 nvidia_compiler: str = {
@@ -72,6 +74,14 @@ def test_install_libfyaml_default():
     spack_install('libfyaml', test_root=False)
 
 
+@pytest.mark.no_tsa  # No uenv on Tsa
+@pytest.mark.no_daint  # No uenv on Daint
+@pytest.mark.no_balfrin  # Does not work as expected
+@pytest.mark.libfyaml
+def test_install_libfyaml_default_uenv():
+    spack_install('libfyaml', test_root=False, uenv=PREPOST)
+
+
 @pytest.mark.libtorch
 def test_install_libtorch_default():
     spack_install('libtorch', test_root=False)
@@ -172,20 +182,22 @@ def icon_env_test(spack_env: str, out_of_source: bool = False):
         check=True,
         shell=True)
 
+    log_filename = sanitized_filename(spack_env)
+
     if out_of_source:
         build_dir = os.path.join(unique_folder, 'build')
         os.makedirs(build_dir, exist_ok=True)
         shutil.copytree(os.path.join(unique_folder, 'config'),
                         os.path.join(build_dir, 'config'))
         unique_folder = build_dir
+        log_filename += '_out_of_source'
 
-    log_filename = sanitized_filename(spack_env) + '_out_of_source'
     log_with_spack('spack install -n -v',
                    'system_test',
                    log_filename,
                    cwd=unique_folder,
                    env=spack_env,
-                   srun=True)
+                   allow_srun=True)
 
     # for out-of-source build we can't run tests because required files
     # like scripts/spack/test.py or scripts/buildbot_script are not synced
@@ -198,7 +210,7 @@ def icon_env_test(spack_env: str, out_of_source: bool = False):
                    log_filename,
                    cwd=unique_folder,
                    env=spack_env,
-                   srun=False)
+                   allow_srun=False)
 
 
 @pytest.mark.no_balfrin  # config file does not exist for this machine