Merge branch 'branch-23.07' into mdd_combining-operators

ci/conda/environments/ci_env.yml

@@ -19,3 +19,4 @@ channels:
     - conda-forge
 dependencies:
     - codecov=2.1
+    - conda-merge>=0.2

ci/conda/environments/dev_env.yml

@@ -1,21 +1,38 @@
+# SPDX-FileCopyrightText: Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Dependencies needed for development environment. Runtime deps are in meta.yml
+name: mrc
 channels:
-  - conda-forge
   - rapidsai
+  - nvidia/label/cuda-11.8.0
   - nvidia
   - rapidsai-nightly
+  - conda-forge
 dependencies:
   - autoconf>=2.69
   - bash-completion
   - benchmark=1.6.0
   - boost-cpp=1.74
   - ccache
   - cmake=3.24
-  - cuda-nvml-dev=11.8
-  - cudatoolkit=11.8
-  - cython>=0.29,<0.30
+  - cuda-toolkit # Version comes from the channel above
+  - cxx-compiler # Sets up the distro versions of our compilers
   - doxygen=1.9.2
+  - flake8
   - flatbuffers=2.0
-  - gcc_linux-64=11.2
   - gcovr=5.0
   - gdb
   - gflags=2.2
@@ -25,7 +42,7 @@ dependencies:
   - graphviz=3.0
   - grpc-cpp=1.48
   - gtest=1.10
-  - gxx_linux-64=11.2
+  - gxx=11.2 # Specifies which versions of GXX and GCC to use
   - isort
   - jinja2=3.0
   - lcov=1.15
@@ -37,19 +54,17 @@ dependencies:
   - nlohmann_json=3.9
   - numactl-libs-cos7-x86_64
   - numpy>=1.21
-  - nvcc_linux-64=11.8
   - pip
   - pkg-config=0.29
   - pybind11-stubgen=0.10
+  - pytest
+  - pytest-timeout
   - python=3.10
-  - scikit-build>=0.12
+  - scikit-build>=0.17
   - spdlog=1.8.5
   - sysroot_linux-64=2.17
   - ucx=1.13
-  - pip:
-    - cython
-    - flake8
-    - numpy==1.21.2
-    - pytest
-    - pytest-timeout
-    - yapf
+  - yapf
+
+  # Remove once `mamba repoquery whoneeds cudatoolkit` is empty. For now, we need to specify a version
+  - cudatoolkit=11.8

ci/conda/recipes/libmrc/build.sh

@@ -51,17 +51,17 @@ export CC=${GCC}
 export CXX=${GXX}
 
 # Common CMake args
-CMAKE_ARGS="-DCMAKE_MESSAGE_CONTEXT_SHOW=ON ${CMAKE_ARGS}"
-CMAKE_ARGS="-DCMAKE_INSTALL_PREFIX=$PREFIX ${CMAKE_ARGS}"
-CMAKE_ARGS="-DCMAKE_INSTALL_LIBDIR=lib ${CMAKE_ARGS}"
 CMAKE_ARGS="-DBUILD_SHARED_LIBS=ON ${CMAKE_ARGS}"
 CMAKE_ARGS="-DCMAKE_BUILD_TYPE=${MRC_BUILD_TYPE} ${CMAKE_ARGS}"
-CMAKE_ARGS="-DMRC_USE_CONDA=ON ${CMAKE_ARGS}"
-CMAKE_ARGS="-DMRC_USE_CCACHE=OFF ${CMAKE_ARGS}"
-CMAKE_ARGS="-DMRC_BUILD_PYTHON=ON ${CMAKE_ARGS}"
 CMAKE_ARGS="-DCMAKE_CUDA_ARCHITECTURES=${CMAKE_CUDA_ARCHITECTURES=-"ALL"} ${CMAKE_ARGS}"
-CMAKE_ARGS="-DPython_EXECUTABLE=${PYTHON} ${CMAKE_ARGS}"
+CMAKE_ARGS="-DCMAKE_INSTALL_LIBDIR=lib ${CMAKE_ARGS}"
+CMAKE_ARGS="-DCMAKE_INSTALL_PREFIX=$PREFIX ${CMAKE_ARGS}"
+CMAKE_ARGS="-DCMAKE_MESSAGE_CONTEXT_SHOW=ON ${CMAKE_ARGS}"
+CMAKE_ARGS="-DMRC_BUILD_PYTHON=ON ${CMAKE_ARGS}"
 CMAKE_ARGS="-DMRC_RAPIDS_VERSION=${rapids_version} ${CMAKE_ARGS}"
+CMAKE_ARGS="-DMRC_USE_CCACHE=OFF ${CMAKE_ARGS}"
+CMAKE_ARGS="-DMRC_USE_CONDA=ON ${CMAKE_ARGS}"
+CMAKE_ARGS="-DPython_EXECUTABLE=${PYTHON} ${CMAKE_ARGS}"
 CMAKE_ARGS="-DUCX_VERSION=${ucx} ${CMAKE_ARGS}"
 
 echo "CC          : ${CC}"

ci/conda/recipes/libmrc/conda_build_config.yaml

@@ -20,7 +20,7 @@ cxx_compiler_version:
   - 11.2
 
 cuda_compiler:
-  - nvcc
+  - cuda-nvcc
 
 cuda_compiler_version:
   - 11.8

ci/conda/recipes/libmrc/meta.yaml

@@ -43,35 +43,37 @@ requirements:
     - autoconf >=2.69
     - ccache
     - cmake >=3.24
+    - cuda-cudart-dev # Needed by CMake to compile a test application
     - libtool
     - ninja
     - numactl-libs-cos7-x86_64
     - pkg-config 0.29.*
     - sysroot_linux-64 >=2.17
   host:
     # Libraries necessary to build. Keep sorted!
-    - libabseil
     - boost-cpp
-    - cuda-nvml-dev {{ cuda_version }}.*
-    - cudatoolkit {{ cuda_version }}.*
-    - cython 0.29.*
+    - cuda-cudart-dev
+    - cuda-nvml-dev
     - doxygen 1.9.2.*
     - flatbuffers 2.0.*
     - gflags
     - glog
     - gmock 1.10.*
     - grpc-cpp
     - gtest 1.10.*
+    - libabseil
     - libhwloc 2.5.*
     - libprotobuf
     - librmm {{ rapids_version }}
     - nlohmann_json 3.9.1
     - pybind11-abi # See: https://conda-forge.org/docs/maintainer/knowledge_base.html#pybind11-abi-constraints
-    - pybind11-stubgen 0.10
+    - pybind11-stubgen 0.10.5
     - python {{ python }}
-    - scikit-build >=0.12
+    - scikit-build >=0.17
     - spdlog 1.8.5
     - ucx
+    # Need to specify cudatoolkit to get correct version. Remove once all libraries migrate to cuda-toolkit
+    - cudatoolkit {{ cuda_version }}.*
 
 outputs:
   - name: libmrc
@@ -92,23 +94,27 @@ outputs:
         - sysroot_linux-64 2.17
       host:
         # Any libraries with weak run_exports need to go here to be added to the run. Keep sorted!
-        - libabseil # Needed for transitive run_exports from grpc-cpp. Does not need a version
         - boost-cpp
-        - flatbuffers 2.0.*
-        - gflags
+        - cuda-cudart # Needed to allow pin_compatible to work
         - glog
         - grpc-cpp
+        - libabseil # Needed for transitive run_exports from grpc-cpp. Does not need a version
         - libhwloc 2.5.*
         - libprotobuf # Needed for transitive run_exports from grpc-cpp. Does not need a version
         - librmm {{ rapids_version }}
         - nlohmann_json 3.9.*
         - ucx
+        # Need to specify cudatoolkit to get correct version. Remove once all libraries migrate to cuda-toolkit
+        - cudatoolkit {{ cuda_version }}.*
       run:
         # Manually add any packages necessary for run that do not have run_exports. Keep sorted!
-        - {{ pin_compatible('flatbuffers', max_pin='x.x')}}
+        - {{ pin_compatible('cuda-cudart', min_pin='x.x', max_pin='x') }}
         - {{ pin_compatible('nlohmann_json', max_pin='x.x')}}
         - {{ pin_compatible('ucx', max_pin='x.x')}}
         - boost-cpp # Needed to use pin_run_as_build
+      run_constrained:
+        # Since we dont explicitly require this but other packages might, constrain the versions
+        - {{ pin_compatible('cudatoolkit', min_pin='x.x', max_pin='x') }}
     test:
       script: test_libmrc.sh
       files:
@@ -121,13 +127,16 @@ outputs:
         - {{ pin_subpackage("mrc", max_pin="x.x") }}
       ignore_run_exports:
         - pypy
+      missing_dso_whitelist:
+        - $RPATH/ld-linux-x86-64.so.2 # From python
+        - $RPATH/libc.so.6 # From python
+        - $RPATH/libpthread.so.0 # Sample modules
       include_recipe: False
     script: move_mrc_files.sh
     requirements:
       host:
         # Only should need libmrc and python. Keep sorted!
         - {{ pin_subpackage('libmrc', exact=True) }}
-        - libabseil # mrc does not require abseil at build time. See https://github.com/conda-forge/arrow-cpp-feedstock/issues/814
         - python {{ python }}
       run:
         - {{ pin_subpackage('libmrc', exact=True) }}
@@ -149,8 +158,8 @@ outputs:
         - cuml {{ rapids_version }}.* # Ensure we can install cuml. This can cause issues solving libabseil
 
 about:
-  home: https://www.nvidia.com/
-  summary: A GPU accelerated streaming data library with python bindings
-  license: Apache-2.0
+  home: https://github.com/nv-morpheus/MRC
   license_family: Apache
   license_file: LICENSE
+  license: Apache-2.0
+  summary: A GPU accelerated streaming data library with python bindings

ci/conda/recipes/run_conda_build.sh

@@ -40,13 +40,6 @@ export CONDA_COMMAND=${CONDA_COMMAND:-"mambabuild"}
 # Get the path to the morpheus git folder
 export MRC_ROOT=${MRC_ROOT:-$(git rev-parse --show-toplevel)}
 
-export CUDA="$(conda list | grep cudatoolkit | egrep -o "[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+")"
-export PYTHON_VER="$(python -c "import sys; print('.'.join(map(str, sys.version_info[:2])))")"
-export CUDA=11.8.0
-echo "CUDA       : ${CUDA}"
-echo "PYTHON_VER : ${PYTHON_VER}"
-echo ""
-
 export PARALLEL_LEVEL=${PARALLEL_LEVEL:-$(nproc)}
 
 # Export variables for the cache
@@ -103,7 +96,7 @@ if hasArg quick; then
 fi
 
 # And default channels
-CONDA_ARGS_ARRAY+=("-c" "rapidsai" "-c" "nvidia" "-c" "conda-forge" "-c" "main")
+CONDA_ARGS_ARRAY+=("-c" "rapidsai" "-c" "nvidia/label/cuda-11.8.0" "-c" "nvidia" "-c" "conda-forge" "-c" "main")
 
 # Set GIT_VERSION to set the project version inside of meta.yaml
 export GIT_VERSION="$(get_version)"

ci/scripts/github/common.sh

@@ -37,6 +37,7 @@ export BUILD_CC=${BUILD_CC:-"gcc"}
 
 export CONDA_ENV_YML="${MRC_ROOT}/ci/conda/environments/dev_env.yml"
 export CONDA_CLANG_ENV_YML="${MRC_ROOT}/ci/conda/environments/clang_env.yml"
+export CONDA_CI_ENV_YML="${MRC_ROOT}/ci/conda/environments/ci_env.yml"
 
 export CMAKE_BUILD_ALL_FEATURES="-DCMAKE_MESSAGE_CONTEXT_SHOW=ON -DMRC_BUILD_BENCHMARKS=ON -DMRC_BUILD_EXAMPLES=ON -DMRC_BUILD_PYTHON=ON -DMRC_BUILD_TESTS=ON -DMRC_USE_CONDA=ON -DMRC_PYTHON_BUILD_STUBS=ON"
 export CMAKE_BUILD_WITH_CODECOV="-DCMAKE_BUILD_TYPE=Debug -DMRC_ENABLE_CODECOV=ON -DMRC_PYTHON_PERFORM_INSTALL:BOOL=ON -DMRC_PYTHON_INPLACE_BUILD:BOOL=ON"
@@ -73,10 +74,33 @@ function print_env_vars() {
 
 function update_conda_env() {
     rapids-logger "Checking for updates to conda env"
-    rapids-mamba-retry env update -n mrc -q --file ${CONDA_ENV_YML}
-    rapids-mamba-retry env update -n mrc -q --file ${CONDA_CLANG_ENV_YML}
+
+    # Deactivate the environment first before updating
     conda deactivate
+
+    # Make sure we have the conda-merge package installed
+    if [[ -z "$(conda list | grep conda-merge)" ]]; then
+        rapids-mamba-retry install -n mrc -c conda-forge "conda-merge>=0.2"
+    fi
+
+    # Create a temp directory which we store the combined environment file in
+    condatmpdir=$(mktemp -d)
+
+    # Merge the environments together so we can use --prune. Otherwise --prune
+    # will clobber the last env update
+    conda run -n mrc --live-stream conda-merge ${CONDA_ENV_YML} ${CONDA_CLANG_ENV_YML} ${CONDA_CI_ENV_YML} > ${condatmpdir}/merged_env.yml
+
+    # Update the conda env with prune remove excess packages (in case one was removed from the env)
+    rapids-mamba-retry env update -n mrc -q --prune --file ${condatmpdir}/merged_env.yml
+
+    # Delete the temp directory
+    rm -rf ${condatmpdir}
+
+    # Finally, reactivate
     conda activate mrc
+
+    rapids-logger "Final Conda Environment"
+    conda list
 }
 
 print_env_vars

cpp/mrc/src/internal/control_plane/server.cpp

@@ -204,7 +204,7 @@ void Server::do_service_await_join()
 /**
  * @brief Stream Acceptor
  *
- * The while loop of this method says active as long as the grpc server is still accepting connections.
+ * The while loop of this method stays active as long as the grpc server is still accepting connections.
  * There are multiple way this can be implemented depending the service requirements, one might choose
  * to preallocate N number of streams and issues them all to the CQ. This is an alternative method which
  * creates a single stream and waits for it to get initialized, then creates another. The current implementation is

cpp/mrc/src/internal/control_plane/server/subscription_manager.hpp

@@ -41,7 +41,7 @@ class Role;
 class ClientInstance;
 
 /**
- * @brief A specialize TaggedManager to synchronize tag and instance_id information across between a collection of
+ * @brief A specialized TaggedIssuer to synchronize tag and instance_id information across between a collection of
  * client-side objects with common linkages, e.g. the Publisher/Subscriber services which form the building blocks for
  * Ingress/EgressPorts use instances of SubscriptionService for Publishers to get control plane updates to the list of
  * Subscribers.

cpp/mrc/src/internal/control_plane/server/tagged_issuer.hpp

@@ -70,9 +70,9 @@ class Tagged
  * This is the primary base class for a control plane server-side stateful service which can be updated by the client
  * and state updates driven independently via the issue_update() method.
  *
- * TaggedManager is not thread-safe or protected in anyway. The global state mutex should protect all TaggedManagers.
+ * TaggedIssuer is not thread-safe or protected in anyway. The global state mutex should protect all TaggedIssuers.
  *
- * In most scenarios, the service side will have a batched updated which will periodically visit each TaggedManager and
+ * In most scenarios, the service side will have a batched updated which will periodically visit each TaggedIssuer and
  * call issue_update(); however, depending on the service request/update message, the call may also require an immediate
  * update.
  */

cpp/mrc/src/public/segment/builder.cpp

@@ -49,14 +49,14 @@ void Builder::init_module(std::shared_ptr<mrc::modules::SegmentModule> smodule)
     VLOG(2) << "Initializing module: " << m_namespace_prefix;
     smodule->m_module_instance_registered_namespace = m_namespace_prefix;
     smodule->initialize(*this);
-    ns_pop();
 
     // TODO(Devin): Maybe a better way to do this with compile time type ledger.
     if (std::dynamic_pointer_cast<modules::PersistentModule>(smodule) != nullptr)
     {
-        VLOG(2) << "Registering persistent module -> '" << smodule->component_prefix() << "'";
+        VLOG(2) << "Registering persistent module -> '" << m_namespace_prefix << "'";
         m_backend.add_module(m_namespace_prefix, smodule);
     }
+    ns_pop();
 }
 
 std::shared_ptr<ObjectProperties> Builder::get_ingress(std::string name, std::type_index type_index)

docs/quickstart/CMakeLists.txt

@@ -39,7 +39,8 @@ rapids_cpm_init()
 
 # Set the option prefix to match the outer project before including. Must be before find_package(mrc)
 set(OPTION_PREFIX "MRC")
-morpheus_utils_python_ensure_loaded()
+
+morpheus_utils_python_configure()
 
 rapids_find_package(mrc REQUIRED)
 rapids_find_package(CUDAToolkit REQUIRED)

python/CMakeLists.txt

@@ -19,7 +19,7 @@ find_package(CUDAToolkit REQUIRED)
 
 
 # Ensure python is configured
-morpheus_utils_python_ensure_loaded()
+morpheus_utils_python_configure()
 
 
 morpheus_utils_print_python_info()

python/setup.py

@@ -14,7 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from setuptools import find_packages
+from setuptools import find_namespace_packages
 from setuptools import setup
 
 import versioneer
@@ -23,7 +23,7 @@
 # - Python package generation ------------------------------------------------
 
 setup(
-    name='mrc',
+    name="mrc",
     description="mrc",
     version=versioneer.get_version(),
     classifiers=[
@@ -32,9 +32,10 @@
     author="NVIDIA Corporation",
     setup_requires=[],
     include_package_data=True,
-    packages=find_packages(include=['mrc', 'mrc.*'], exclude=['tests']),
+    packages=find_namespace_packages(include=["mrc*"], exclude=["tests", "mrc.core.segment.module_definitions"]),
     package_data={
         "mrc": ["_pymrc/*.so"]  # Add the pymrc library for the root package
     },
     license="Apache",
-    cmdclass=versioneer.get_cmdclass())
+    cmdclass=versioneer.get_cmdclass(),
+    zip_safe=False)