Merge remote-tracking branch 'upstream/master'

Conflicts:
	site_scons/prereq_tools/base.py

.github/workflows/spelling.yml

@@ -13,7 +13,7 @@ jobs:
       - name: Run check
         uses: codespell-project/actions-codespell@master
         with:
-          skip: ./src/control/vendor,./.git
+          skip: ./src/control/vendor,./src/control/go.sum,./.git
           ignore_words_file: ci/codespell.ignores
           builtin: clear,rare,informal,names,en-GB_to_en-US
       - name: Check DAOS logging macro use.

.rpmignore

@@ -3,28 +3,22 @@
 # but should not be included in the current release
 #
 
-centos7/daos-client-tests-openmpi*.rpm
-centos7/daos-firmware*.rpm
-centos7/daos-mofed*.rpm
-centos7/daos-serialize*.rpm
-centos7/daos-server-tests-openmpi*.rpm
-centos7/daos-tests-internal*.rpm
-centos7/ucx*.rpm
-
-el8/daos-client-tests-openmpi*.rpm
+el8/daos-*tests*.rpm
 el8/daos-firmware*.rpm
 el8/daos-mofed*.rpm
 el8/daos-serialize*.rpm
-el8/daos-server-tests-openmpi*.rpm
-el8/daos-tests-internal*.rpm
 el8/ucx*.rpm
 
-leap15/daos-client-tests-openmpi*.rpm
+el9/daos-*tests*.rpm
+el9/daos-firmware*.rpm
+el9/daos-mofed*.rpm
+el9/daos-serialize*.rpm
+el9/ucx*.rpm
+
+leap15/daos-*tests*.rpm
 leap15/daos-firmware*.rpm
 leap15/daos-mofed*.rpm
 leap15/daos-serialize*.rpm
-leap15/daos-server-tests-openmpi*.rpm
-leap15/daos-tests-internal*.rpm
 leap15/openucx*.rpm
 leap15/ucx*.rpm
 leap15/*protobuf-c*.rpm

Jenkinsfile

@@ -74,6 +74,11 @@ void job_step_update(def value) {
 Map nlt_test() {
     // groovylint-disable-next-line NoJavaUtilDate
     Date startDate = new Date()
+    try {
+        unstash('nltr')
+    } catch (e) {
+        print 'Unstash failed, results from NLT stage will not be included'
+    }
     sh label: 'Fault injection testing using NLT',
        script: './ci/docker_nlt.sh --class-name el8.fault-injection fi'
     List filesList = []
@@ -1104,7 +1109,6 @@ pipeline {
                             sconsBuild(parallel_build: true,
                                        scons_args: 'PREFIX=/opt/daos TARGET_TYPE=release BUILD_TYPE=debug',
                                        build_deps: 'no'))
-                        unstash('nltr')
                         job_step_update(nlt_test())
                         recordCoverage(tools: [[parser: 'COBERTURA', pattern:'nltr.xml']],
                                        skipPublishingChecks: true,

SConstruct

@@ -363,7 +363,7 @@ MINIMAL_ENV = ('HOME', 'TERM', 'SSH_AUTH_SOCK', 'http_proxy', 'https_proxy', 'PK
 
 # Environment variables that are also kept when LD_PRELOAD is set.
 PRELOAD_ENV = ('LD_PRELOAD', 'D_LOG_FILE', 'DAOS_AGENT_DRPC_DIR', 'D_LOG_MASK', 'DD_MASK',
-               'DD_SUBSYS')
+               'DD_SUBSYS', 'D_IL_MAX_EQ')
 
 
 def scons():

ci/rpm/build_unsuccessful.sh

@@ -8,7 +8,7 @@ mydir="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
 ci_envs="$mydir/../parse_ci_envs.sh"
 if [ -e "${ci_envs}" ]; then
   # at some point we want to use: shellcheck source=ci/parse_ci_envs.sh
-  # shellcheck disable=SC1091
+  # shellcheck disable=SC1091,SC1090
   source "${ci_envs}"
 fi
 
@@ -25,6 +25,9 @@ if [ -d /var/cache/pbuilder/ ]; then
     exit 0
 fi
 
+rpm -q mock
+mock --debug-config
+
 mockroot="/var/lib/mock/$CHROOT_NAME"
 cat "$mockroot"/result/{root,build}.log 2>/dev/null || true
 

ci/test_files_to_stash.txt

@@ -1,5 +1,6 @@
 build/*/*/src/tests/ftest/cart/utest/test_linkage,
 build/*/*/src/tests/ftest/cart/utest/utest_hlc,
+build/*/*/src/tests/ftest/cart/utest/utest_protocol,
 build/*/*/src/tests/ftest/cart/utest/utest_swim,
 build/*/*/src/gurt/tests/test_gurt,
 build/*/*/src/gurt/tests/test_gurt_telem_producer,

docs/admin/administration.md

@@ -960,3 +960,28 @@ DAOS v2.2 client connections to pools which were created by DAOS v2.4
 will be rejected. DAOS v2.4 client should work with DAOS v2.4 and DAOS v2.2
 server. To upgrade all pools to latest format after software upgrade, run
 `dmg pool upgrade <pool>`
+
+### Interoperability Matrix
+
+The following table is intended to visually depict the interoperability
+policies for all major components in a DAOS system.
+
+
+||Server<br>(daos_server)|Engine<br>(daos_engine)|Agent<br>(daos_agent)|Client<br>(libdaos)|Admin<br>(dmg)|
+|:---|:---:|:---:|:---:|:---:|:---:|
+|Server|x.y.z|x.y.z|x.(y±1)|n/a|x.y|
+|Engine|x.y.z|x.y.z|n/a|x.(y±1)|n/a|
+|Agent|x.(y±1)|n/a|n/a|x.y.z|n/a|
+|Client|n/a|x.(y±1)|x.y.z|n/a|n/a|
+|Admin|x.y|n/a|n/a|n/a|n/a|
+
+Key:
+  * x.y.z: Major.Minor.Patch must be equal
+  * x.y: Major.Minor must be equal
+  * x.(y±1): Major must be equal, Minor must be equal or -1/+1 release version
+  * n/a: Components do not communicate
+
+Examples:
+  * daos_server 2.4.0 is only compatible with daos_engine 2.4.0
+  * daos_agent 2.6.0 is compatible with daos_server 2.4.0 (2.5 is a development version)
+  * dmg 2.4.1 is compatible with daos_server 2.4.0

docs/admin/tiering_uns.md

@@ -31,7 +31,8 @@ The current state of work can be summarized as follows :
     pools/containers as `<pool-uuid>/<container-uuid>` relative paths.
 
 -   `daos` foreign support is enabled at mount time with the `symlink=` option
-    present or dynamically, through the `llite.*.daos_enable` setting.
+    present or dynamically, through the `llite.*.foreign_symlink_enable` Lustre
+    dynamic tuneable.
 
 ### Building and using a DAOS-aware Lustre version
 

docs/dev/contributing.md

@@ -13,7 +13,9 @@ features, offer to add a feature, or just begin a dialog about DAOS:
 ## Coding Rules
 
 Please check the [coding conventions](https://wiki.daos.io/spaces/DC/pages/4836655701/Coding+Rules)
-for code contribution.
+for code contribution.  See [githooks](../../utils/githooks/README.md) for information about setting up
+some automated checks on commit to help with adherence to the rules.
+
 
 ## Commit Comments
 

mkdocs.yml

@@ -1,5 +1,5 @@
 # Project Information
-site_name: DAOS v2.5 - master
+site_name: DAOS v2.5
 site_description: Distributed Asynchronous Object Storage
 site_author: DAOS Project
 

site_scons/components/__init__.py

@@ -58,7 +58,7 @@ def check(self, name):
             self.installed.append(name)
             return True
 
-        if not GetOption('help'):
+        if not GetOption('help') and not GetOption('silent'):
             print(f'Using build version of {name}')
         self.not_installed.append(name)
         return False
@@ -90,12 +90,14 @@ def check(reqs, name, built_str, installed_str=""):
 
 def ofi_config(config):
     """Check ofi version"""
-    print('Checking for libfabric > 1.11...', end=' ')
+    if not GetOption('silent'):
+        print('Checking for libfabric > 1.11...', end=' ')
     code = """#include <rdma/fabric.h>
 _Static_assert(FI_MAJOR_VERSION == 1 && FI_MINOR_VERSION >= 11,
                "libfabric must be >= 1.11");"""
     rc = config.TryCompile(code, ".c")
-    print('yes' if rc else 'no')
+    if not GetOption('silent'):
+        print('yes' if rc else 'no')
     return rc
 
 
@@ -132,6 +134,7 @@ def define_mercury(reqs):
                 libs=['fabric'],
                 config_cb=ofi_config,
                 headers=['rdma/fabric.h'],
+                pkgconfig='libfabric',
                 package='libfabric-devel' if inst(reqs, 'ofi') else None,
                 patch_rpath=['lib'],
                 build_env={'CFLAGS': "-fstack-usage"})
@@ -184,12 +187,6 @@ def define_mercury(reqs):
     else:
         mercury_build.append('-DMERCURY_ENABLE_DEBUG:BOOL=OFF')
 
-    mercury_build.extend(check(reqs,
-                               'ofi',
-                               ['-DOFI_INCLUDE_DIR:PATH=$OFI_PREFIX/include',
-                                '-DOFI_LIBRARY:FILEPATH=$OFI_PREFIX/lib/libfabric.so'],
-                               []))
-
     reqs.define('mercury',
                 retriever=GitRepoRetriever('https://github.com/mercury-hpc/mercury.git', True),
                 commands=[mercury_build,

site_scons/env_modules.py

@@ -36,14 +36,15 @@ class _env_module():  # pylint: disable=invalid-name
                 "openmpi": ['mpi/mlnx_openmpi-x86_64', 'mpi/openmpi3-x86_64',
                             'gnu-openmpi', 'mpi/openmpi-x86_64']}
 
-    def __init__(self):
+    def __init__(self, silent=False):
         """Load Modules for initializing environment variables"""
         # Leap 15's lmod-lua doesn't include the usual module path
         # in it's MODULEPATH, for some unknown reason
         os.environ["MODULEPATH"] = ":".join([os.path.join(os.sep, "usr", "share", "modules"),
                                              os.path.join(os.sep, "usr", "share", "modulefiles"),
                                              os.path.join(os.sep, "etc", "modulefiles")]
                                             + os.environ.get("MODULEPATH", "").split(":"))
+        self._silent = silent
         self._module_load = self._init_mpi_module()
 
     def _module_func(self, command, *arguments):  # pylint: disable=no-self-use
@@ -56,7 +57,8 @@ def _module_func(self, command, *arguments):  # pylint: disable=no-self-use
 
         # pylint: disable=consider-using-with
         try:
-            print(f"Going to run {cmd}")
+            if not self._silent:
+                print(' '.join(cmd))
             proc = Popen(cmd, stdout=PIPE, stderr=PIPE)
         except OSError as error:
             if error.errno == errno.ENOENT:
@@ -81,11 +83,11 @@ def _module_func(self, command, *arguments):  # pylint: disable=no-self-use
         # return _mlstatus, stderr.decode() # pylint: disable=undefined-variable
 
     def _init_mpi_module(self):
-        """init mpi module function"""
+        """Init mpi module function"""
         return self._mpi_module
 
     def _mpi_module(self, mpi):
-        """attempt to load the requested module"""
+        """Attempt to load the requested module"""
         load = []
         unload = []
 
@@ -109,16 +111,17 @@ def _mpi_module(self, mpi):
                 self._module_func('unload', to_unload)
 
         for to_load in load:
-            print(f"Trying to load {to_load}")
-            if self._module_func('is-avail', to_load)[0] and \
-               self._module_func('load', to_load)[0]:
-                print(f'Loaded {to_load}')
+            if not self._silent:
+                print(f"Trying to load {to_load}")
+            if self._module_func('is-avail', to_load)[0] and self._module_func('load', to_load)[0]:
+                if not self._silent:
+                    print(f'Loaded {to_load}')
                 return True
 
         return False
 
     def _mpi_module_old(self, mpi):
-        """attempt to load the requested module"""
+        """Attempt to load the requested module"""
         load = []
         for key, value in self._mpi_map.items():
             if key == mpi:
@@ -162,7 +165,7 @@ def load_mpi(self, mpi):
         return True
 
     def show_avail(self):
-        """list available modules"""
+        """List available modules"""
         try:
             status, output = self._module_func('avail')
             if not status:
@@ -172,12 +175,12 @@ def show_avail(self):
         return output
 
     def get_map(self, key):
-        """return the mpi map"""
+        """Return the mpi map"""
         return self._mpi_map[key]
 
 
-def load_mpi(mpi):
-    """global function to load MPI into os.environ"""
+def load_mpi(mpi, silent=False):
+    """Global function to load MPI into os.environ"""
     # On Ubuntu, MPI stacks use alternatives and need root to change their
     # pointer, so just verify that the desired MPI is loaded
     if distro.id() == "ubuntu":
@@ -201,19 +204,19 @@ def load_mpi(mpi):
         return False
 
     if _env_module.env_module_init is None:
-        _env_module.env_module_init = _env_module()
+        _env_module.env_module_init = _env_module(silent)
     return _env_module.env_module_init.load_mpi(mpi)
 
 
 def show_avail():
-    """global function to show the available modules"""
+    """Global function to show the available modules"""
     if _env_module.env_module_init is None:
         _env_module.env_module_init = _env_module()
     return _env_module.env_module_init.show_avail()
 
 
 def get_module_list(key):
-    """global function to show the modules that map to a key"""
+    """Global function to show the modules that map to a key"""
     if _env_module.env_module_init is None:
         _env_module.env_module_init = _env_module()
     return _env_module.env_module_init.get_map(key)

site_scons/prereq_tools/base.py

@@ -1155,6 +1155,11 @@ def _parse_config(self, env, opts):
 
         return
 
+    def _print(self, msg):
+        if GetOption('silent'):
+            return
+        print(msg)
+
     def has_missing_targets(self, env):
         """Check for expected build targets (e.g. libraries or headers)"""
         # pylint: disable=too-many-return-statements
@@ -1184,6 +1189,7 @@ def has_missing_targets(self, env):
         prGreen(f"Checking targets for component '{self.name}'")
         # if self.name == 'argobots':
         #     return False
+        # self._print(f"Checking targets for component '{self.name}'")
 
         config = env.Configure()
         config_cb = self.key_words.get("config_cb", None)
@@ -1276,7 +1282,6 @@ def configure(self):
 
     def set_environment(self, env, needed_libs):
         """Modify the specified construction environment to build with the external component"""
-
         if self.skip_arch:
             return
 

site_scons/site_tools/compiler_setup.py

@@ -39,8 +39,9 @@ def _base_setup(env):
     compiler = env['CC']
 
     build_type = env['BUILD_TYPE']
-    print(f'Setting up compile environment for {compiler}')
-    print(f"Build type is '{build_type}'")
+    if not GetOption('silent'):
+        print(f'Setting up compile environment for {compiler}')
+        print(f"Build type is '{build_type}'")
 
     prev_compiler = env.get('BSETUP', False)
     if prev_compiler:

site_scons/site_tools/daos_builder.py

@@ -224,6 +224,10 @@ def _configure_mpi(self):
     if GetOption('help'):
         return None
 
+    def _print(msg):
+        if not GetOption('silent'):
+            print(msg)
+
     env = self.Clone()
 
     env['CXX'] = None
@@ -233,13 +237,13 @@ def _configure_mpi(self):
         return env
 
     for mpi in ['openmpi', 'mpich']:
-        if not load_mpi(mpi):
+        if not load_mpi(mpi, GetOption('silent')):
             continue
         if _find_mpicc(env):
-            print(f'{mpi} is installed')
+            _print(f'{mpi} is installed')
             return env
-        print(f'No {mpi} installed and/or loaded')
-    print("No MPI installed")
+        _print(f'No {mpi} installed and/or loaded')
+    _print("No MPI installed")
     return None
 
 

src/SConscript

@@ -49,7 +49,8 @@ def read_and_save_version(env):
                       '@Template for @': ''}
 
         out = env.Substfile(tmpl_hdr_in, SUBST_DICT=subst_dict)
-        print(f'generated daos version header file: {out[0].abspath}')
+        if not GetOption('silent'):
+            print(f'generated daos version header file: {out[0].abspath}')
 
         return version