Merge pull request #507 from nv-morpheus/branch-24.10

[RELEASE] MRC v24.10.00

.devcontainer/conda/Dockerfile

@@ -13,6 +13,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM rapidsai/devcontainers:23.04-cuda12.1-mambaforge-ubuntu22.04 AS base
+FROM rapidsai/devcontainers:24.12-cuda12.1-mambaforge-ubuntu22.04 AS base
 
 ENV PATH="${PATH}:/workspaces/mrc/.devcontainer/bin"

.devcontainer/conda/devcontainer.json

@@ -35,7 +35,7 @@
         "MRC_ROOT": "${containerWorkspaceFolder}",
         "DEFAULT_CONDA_ENV": "mrc",
         "MAMBA_NO_BANNER": "1",
-        "VAULT_HOST": "https://vault.ops.k8s.rapids.ai"
+        "AWS_ROLE_ARN": "arn:aws:iam::279114543810:role/nv-gha-token-sccache-devs"
     },
     "initializeCommand": [ "${localWorkspaceFolder}/.devcontainer/conda/initialize-command.sh" ],
     "remoteUser": "coder",

.devcontainer/opt/mrc/bin/post-attach-command.sh

@@ -28,6 +28,6 @@ sed -ri "s/conda activate base/conda activate $ENV_NAME/g" ~/.bashrc;
 
 if conda_env_find "${ENV_NAME}" ; \
 
-then mamba env update --name ${ENV_NAME} -f ${MRC_ROOT}/conda/environments/all_cuda-121_arch-x86_64.yaml --prune; \
-else mamba env create --name ${ENV_NAME} -f ${MRC_ROOT}/conda/environments/all_cuda-121_arch-x86_64.yaml; \
+then mamba env update --name ${ENV_NAME} -f ${MRC_ROOT}/conda/environments/all_cuda-125_arch-x86_64.yaml --prune; \
+else mamba env create --name ${ENV_NAME} -f ${MRC_ROOT}/conda/environments/all_cuda-125_arch-x86_64.yaml; \
 fi

.devcontainer/opt/mrc/conda/Dockerfile

@@ -13,6 +13,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM rapidsai/devcontainers:23.04-cuda12.1-mambaforge-ubuntu22.04 AS base
+FROM rapidsai/devcontainers:24.12-cuda12.1-mambaforge-ubuntu22.04 AS base
 
 ENV PATH="${PATH}:/workspaces/mrc/.devcontainer/bin"

.devcontainer/opt/mrc/conda/devcontainer.json

@@ -35,7 +35,7 @@
         "MRC_ROOT": "${containerWorkspaceFolder}",
         "DEFAULT_CONDA_ENV": "mrc",
         "MAMBA_NO_BANNER": "1",
-        "VAULT_HOST": "https://vault.ops.k8s.rapids.ai"
+        "AWS_ROLE_ARN": "arn:aws:iam::279114543810:role/nv-gha-token-sccache-devs"
     },
     "initializeCommand": [ "${localWorkspaceFolder}/.devcontainer/initialize-command.sh" ],
     "remoteUser": "coder",

.github/workflows/pr.yaml

@@ -49,7 +49,7 @@ jobs:
       - prepare
       - ci_pipe
     secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.02
+    uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.10
   prepare:
     name: Prepare
     runs-on: ubuntu-latest
@@ -58,7 +58,7 @@ jobs:
     steps:
       - name: Get PR Info
         id: get-pr-info
-        uses: rapidsai/shared-action-workflows/get-pr-info@branch-23.08
+        uses: nv-gha-runners/get-pr-info@main
         if: ${{ startsWith(github.ref_name, 'pull-request/') }}
     outputs:
       is_pr: ${{ startsWith(github.ref_name, 'pull-request/') }}
@@ -72,7 +72,7 @@ jobs:
     needs: [prepare]
     if: ${{ !fromJSON(needs.prepare.outputs.has_skip_ci_label) && fromJSON(needs.prepare.outputs.is_pr )}}
     secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-24.02
+    uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-24.10
     with:
       enable_check_generated_files: false
 
@@ -89,9 +89,9 @@ jobs:
       # Update conda package only for non PR branches. Use 'main' for main branch and 'dev' for all other branches
       conda_upload_label: ${{ !fromJSON(needs.prepare.outputs.is_pr) && (fromJSON(needs.prepare.outputs.is_main_branch) && 'main' || 'dev') || '' }}
       # Build container
-      container: nvcr.io/ea-nvidia-morpheus/morpheus:mrc-ci-build-240214
+      container: nvcr.io/ea-nvidia-morpheus/morpheus:mrc-ci-build-241002
       # Test container
-      test_container: nvcr.io/ea-nvidia-morpheus/morpheus:mrc-ci-test-240214
+      test_container: nvcr.io/ea-nvidia-morpheus/morpheus:mrc-ci-test-241002
       # Info about the PR. Empty for non PR branches. Useful for extracting PR number, title, etc.
       pr_info: ${{ needs.prepare.outputs.pr_info }}
     secrets:

.gitmodules

@@ -1,4 +1,4 @@
 [submodule "morpheus_utils"]
 	path = external/utilities
 	url = https://github.com/nv-morpheus/utilities.git
-	branch = branch-24.06
+	branch = branch-24.10

CHANGELOG.md

@@ -15,6 +15,35 @@ See the License for the specific language governing permissions and
 limitations under the License.
 -->
 
+# MRC 24.10.00 (01 Nov 2024)
+
+## 🐛 Bug Fixes
+
+- fix build interface includes ([#514](https://github.com/nv-morpheus/MRC/pull/514)) [@cwharris](https://github.com/cwharris)
+- Fix CMake issue which installed headers incorrectly during conda build ([#511](https://github.com/nv-morpheus/MRC/pull/511)) [@mdemoret-nv](https://github.com/mdemoret-nv)
+- Adding environment variable to allow skip NUMA node check ([#505](https://github.com/nv-morpheus/MRC/pull/505)) [@mdemoret-nv](https://github.com/mdemoret-nv)
+- Fix get-pr-info gha reference ([#500](https://github.com/nv-morpheus/MRC/pull/500)) [@cwharris](https://github.com/cwharris)
+- Stop a python source once the subscriber is no longer subscribed ([#493](https://github.com/nv-morpheus/MRC/pull/493)) [@dagardner-nv](https://github.com/dagardner-nv)
+- Ensure proper initialization of `CMAKE_INSTALL_PREFIX` if needed ([#485](https://github.com/nv-morpheus/MRC/pull/485)) [@dagardner-nv](https://github.com/dagardner-nv)
+- IWYU fixes ([#517](https://github.com/nv-morpheus/MRC/pull/517)) [@dagardner-nv](https://github.com/dagardner-nv)
+
+## 📖 Documentation
+
+- Fix Incorrect docstring in `mrc.core.coro` ([#503](https://github.com/nv-morpheus/MRC/pull/503)) [@dagardner-nv](https://github.com/dagardner-nv)
+
+## 🚀 New Features
+
+- Add Router nodes to support single input, multi output routing. ([#502](https://github.com/nv-morpheus/MRC/pull/502)) [@mdemoret-nv](https://github.com/mdemoret-nv)
+- Update to RAPIDS 24.10 ([#494](https://github.com/nv-morpheus/MRC/pull/494)) [@cwharris](https://github.com/cwharris)
+
+## 🛠️ Improvements
+
+- devcontainer: replace VAULT_HOST with AWS_ROLE_ARN ([#506](https://github.com/nv-morpheus/MRC/pull/506)) [@jjacobelli](https://github.com/jjacobelli)
+- Pass a `mrc.Subscription` object to sources rather than a `mrc.Subscriber` ([#499](https://github.com/nv-morpheus/MRC/pull/499)) [@dagardner-nv](https://github.com/dagardner-nv)
+- Change `LOG(WARNING)` to `VLOG(1)` when no GPUs are detected ([#497](https://github.com/nv-morpheus/MRC/pull/497)) [@dagardner-nv](https://github.com/dagardner-nv)
+- Define a Python source which receives a reference to a subscriber ([#496](https://github.com/nv-morpheus/MRC/pull/496)) [@dagardner-nv](https://github.com/dagardner-nv)
+
+
 # MRC 24.06.00 (03 Jul 2024)
 
 ## 🚀 New Features

CMakeLists.txt

@@ -37,7 +37,7 @@ option(MRC_USE_CONDA "Enables finding dependencies via conda. All dependencies m
   environment" ON)
 option(MRC_USE_IWYU "Enable running include-what-you-use as part of the build process" OFF)
 
-set(MRC_RAPIDS_VERSION "24.02" CACHE STRING "Which version of RAPIDS to build for. Sets default versions for RAPIDS CMake and RMM.")
+set(MRC_RAPIDS_VERSION "24.10" CACHE STRING "Which version of RAPIDS to build for. Sets default versions for RAPIDS CMake and RMM.")
 
 set(MRC_CACHE_DIR "${CMAKE_SOURCE_DIR}/.cache" CACHE PATH "Directory to contain all CPM and CCache data")
 mark_as_advanced(MRC_CACHE_DIR)
@@ -79,11 +79,11 @@ morpheus_utils_initialize_package_manager(
 morpheus_utils_initialize_cuda_arch(mrc)
 
 project(mrc
-  VERSION 24.06.00
+  VERSION 24.10.00
   LANGUAGES C CXX
 )
 
-rapids_cmake_write_version_file(${CMAKE_BINARY_DIR}/autogenerated/include/mrc/version.hpp)
+morpheus_utils_initialize_install_prefix(MRC_USE_CONDA)
 
 # Delay enabling CUDA until after we have determined our CXX compiler
 if(NOT DEFINED CMAKE_CUDA_HOST_COMPILER)
@@ -178,6 +178,47 @@ if(MRC_BUILD_DOCS)
   add_subdirectory(docs)
 endif()
 
+# ##################################################################################################
+# - install export ---------------------------------------------------------------------------------
+
+set(doc_string
+  [=[
+Provide targets for mrc.
+]=])
+
+set(code_string "")
+
+set(rapids_project_version_compat SameMinorVersion)
+
+# Need to explicitly set VERSION ${PROJECT_VERSION} here since rapids_cmake gets
+# confused with the `RAPIDS_VERSION` variable we use
+rapids_export(INSTALL ${PROJECT_NAME}
+  EXPORT_SET ${PROJECT_NAME}-exports
+  GLOBAL_TARGETS libmrc pymrc
+  COMPONENTS python
+  COMPONENTS_EXPORT_SET ${PROJECT_NAME}-python-exports
+  VERSION ${PROJECT_VERSION}
+  NAMESPACE mrc::
+  DOCUMENTATION doc_string
+  FINAL_CODE_BLOCK code_string
+)
+
+# ##################################################################################################
+# - build export -----------------------------------------------------------------------------------
+rapids_export(BUILD ${PROJECT_NAME}
+  EXPORT_SET ${PROJECT_NAME}-exports
+  GLOBAL_TARGETS libmrc pymrc
+  COMPONENTS python
+  COMPONENTS_EXPORT_SET ${PROJECT_NAME}-python-exports
+  VERSION ${PROJECT_VERSION}
+  LANGUAGES C CXX CUDA
+  NAMESPACE mrc::
+  DOCUMENTATION doc_string
+  FINAL_CODE_BLOCK code_string
+)
+
+# ##################################################################################################
+# - debug info -------------------------------------------------------------------------------------
 if (MRC_ENABLE_DEBUG_INFO)
   morpheus_utils_print_all_targets()
 

CONTRIBUTING.md

@@ -85,7 +85,7 @@ cd $MRC_ROOT
 #### Create MRC Conda environment
 ```bash
 # note: `mamba` may be used in place of `conda` for better performance.
-conda env create -n mrc --file $MRC_ROOT/conda/environments/all_cuda-121_arch-x86_64.yaml
+conda env create -n mrc --file $MRC_ROOT/conda/environments/all_cuda-125_arch-x86_64.yaml
 conda activate mrc
 ```
 #### Build MRC

Dockerfile

@@ -17,7 +17,7 @@
 
 
 ARG FROM_IMAGE="rapidsai/ci-conda"
-ARG CUDA_VER=12.1.1
+ARG CUDA_VER=12.5.1
 ARG LINUX_DISTRO=ubuntu
 ARG LINUX_VER=22.04
 ARG PYTHON_VER=3.10
@@ -45,13 +45,13 @@ RUN useradd --uid $USER_UID --gid $USER_GID -m $USERNAME && \
     echo $USERNAME ALL=\(root\) NOPASSWD:ALL > /etc/sudoers.d/$USERNAME && \
     chmod 0440 /etc/sudoers.d/$USERNAME
 
-COPY ./conda/environments/all_cuda-121_arch-x86_64.yaml /opt/mrc/conda/environments/all_cuda-121_arch-x86_64.yaml
+COPY ./conda/environments/all_cuda-125_arch-x86_64.yaml /opt/mrc/conda/environments/all_cuda-125_arch-x86_64.yaml
 
 RUN --mount=type=cache,target=/opt/conda/pkgs,sharing=locked \
     echo "create env: ${PROJ_NAME}" && \
     sudo -g conda -u $USERNAME \
     CONDA_ALWAYS_YES=true \
-    /opt/conda/bin/mamba env create -q -n ${PROJ_NAME} --file /opt/mrc/conda/environments/all_cuda-121_arch-x86_64.yaml && \
+    /opt/conda/bin/mamba env create -q -n ${PROJ_NAME} --file /opt/mrc/conda/environments/all_cuda-125_arch-x86_64.yaml && \
     chmod -R a+rwX /opt/conda && \
     rm -rf /tmp/conda
 

README.md

@@ -118,7 +118,7 @@ cd $MRC_ROOT
 #### Create MRC Conda Environment
 ```bash
 # note: `mamba` may be used in place of `conda` for better performance.
-conda env create -n mrc-dev --file $MRC_ROOT/conda/environments/all_cuda-121_arch-x86_64.yaml
+conda env create -n mrc-dev --file $MRC_ROOT/conda/environments/all_cuda-125_arch-x86_64.yaml
 conda activate mrc-dev
 ```
 <!-- omit in toc -->

ci/conda/README.md

@@ -4,7 +4,7 @@ To build the Conda packages, it's recommended to run the provided scripts from a
 
 ```bash
 cd ${MRC_ROOT}
-docker buildx build --target developement -t mrc-conda-build .
+docker buildx build --target development -t mrc-conda-build .
 ```
 
 This will create the image `mrc-conda-build` that can be used to build MRC conda packages. When running this container, is recommended to set the environment variable `CONDA_PKGS_DIRS` to a path mounted on the host to speed up the build process. Without this variable set, the packages needed during the build will need to be re-downloaded each time the container is run.
@@ -16,14 +16,14 @@ To build and save the MRC conda package, run the following:
 ```bash
 docker run --rm -ti -v $PWD:/work \
    -e CONDA_PKGS_DIRS=/work/.cache/conda_pkgs \
-   -e CONDA_ARGS="--output-folder=/work/.conda-bld" \
+   -e CONDA_ARGS="--output-folder=/work/.tmp/.conda-bld" \
    mrc-conda-build ./ci/conda/recipes/run_conda_build.sh
 ```
 
-This will save the conda packages to `${MRC_ROOT}/.conda-bld`. To install from this location, use the following:
+This will save the conda packages to `${MRC_ROOT}/.tmp/.conda-bld`. To install from this location, use the following:
 
 ```bash
-conda install -c file://${MRC_ROOT}/.conda-bld mrc
+conda install -c file://${MRC_ROOT}/.tmp/.conda-bld mrc
 ```
 
 ## Uploading the Conda Package

ci/conda/recipes/libmrc/conda_build_config.yaml

@@ -14,20 +14,20 @@
 # limitations under the License.
 
 c_compiler_version:
-  - 11.2
+  - 12.1
 
 cxx_compiler_version:
-  - 11.2
+  - 12.1
 
 cuda_compiler:
   - cuda-nvcc
 
 cuda_compiler_version:
-  - 12.1
+  - 12.5
 
 python:
   - 3.10
 
 # Setup the dependencies to build with multiple versions of RAPIDS
 rapids_version: # Keep around compatibility with current version -2
-  - 24.02
+  - 24.10

ci/conda/recipes/libmrc/meta.yaml

@@ -15,7 +15,7 @@
 
 {% set version = environ.get('GIT_VERSION', '0.0.0.dev').lstrip('v') + environ.get('VERSION_SUFFIX', '') %}
 {% set py_version = environ.get('CONDA_PY', '3.10') %}
-{% set cuda_version = '.'.join(environ.get('CUDA', '12.1').split('.')[:2]) %}
+{% set cuda_version = '.'.join(environ.get('CUDA', '12.5').split('.')[:2]) %}
 
 package:
   name: libmrc-split
@@ -42,9 +42,9 @@ requirements:
     - cmake =3.27
     - libtool
     - ninja =1.11
-    - numactl-libs-cos7-x86_64
+    - numactl =2.0.18
     - pkg-config =0.29
-    - sysroot_linux-64 >=2.17
+    - sysroot_linux-64 >=2.28
   host:
     # Libraries necessary to build. Keep sorted!
     - boost-cpp =1.84
@@ -53,8 +53,8 @@ requirements:
     - cuda-nvrtc-dev {{ cuda_version }}.*
     - cuda-version {{ cuda_version }}.*
     - doxygen 1.10.0
-    - glog =0.6
-    - libgrpc =1.59
+    - glog>=0.7.1,<0.8
+    - libgrpc =1.62.2
     - gtest =1.14
     - libhwloc =2.9.2
     - librmm {{ rapids_version }}
@@ -80,14 +80,14 @@ outputs:
         - {{ compiler("cuda") }}
         - {{ compiler("cxx") }}
         - cmake =3.27
-        - numactl-libs-cos7-x86_64
-        - sysroot_linux-64 =2.17
+        - numactl =2.0.18
+        - sysroot_linux-64 >=2.28
       host:
         # Any libraries with weak run_exports need to go here to be added to the run. Keep sorted!
         - boost-cpp =1.84
         - cuda-version # Needed to allow pin_compatible to work
-        - glog =0.6
-        - libgrpc =1.59
+        - glog>=0.7.1,<0.8
+        - libgrpc =1.62.2
         - libhwloc =2.9.2
         - librmm {{ rapids_version }}
         - nlohmann_json =3.11

ci/conda/recipes/run_conda_build.sh

@@ -95,7 +95,7 @@ fi
 # Choose default variants
 if hasArg quick; then
    # For quick build, just do most recent version of rapids
-   CONDA_ARGS_ARRAY+=("--variants" "{rapids_version: 24.02}")
+   CONDA_ARGS_ARRAY+=("--variants" "{rapids_version: 24.10}")
 fi
 
 # And default channels (should match dependencies.yaml)

ci/iwyu/mappings.imp

@@ -3,6 +3,7 @@
 ## Include mappings
 
 # stdlib
+{ "include": [ "<bits/chrono.h>", private, "<chrono>", "public" ] },
 { "include": [ "<bits/cxxabi_forced.h>", private, "<mutex>", "public" ] },
 { "include": [ "<bits/cxxabi_forced.h>", private, "<vector>", "public" ] },
 { "include": [ "<bits/this_thread_sleep.h>", private, "<thread>", "public" ] },

ci/scripts/cpp_checks.sh

@@ -80,9 +80,8 @@ if [[ -n "${MRC_MODIFIED_FILES}" ]]; then
 
    # Include What You Use
    if [[ "${SKIP_IWYU}" == "" ]]; then
-      # Remove .h, .hpp, and .cu files from the modified list
       shopt -s extglob
-      IWYU_MODIFIED_FILES=( "${MRC_MODIFIED_FILES[@]/*.@(h|hpp|cu)/}" )
+      IWYU_MODIFIED_FILES=( "${MRC_MODIFIED_FILES[@]}" )
 
       if [[ -n "${IWYU_MODIFIED_FILES}" ]]; then
          # Get the list of compiled files relative to this directory

ci/scripts/github/common.sh

@@ -35,7 +35,7 @@ id
 export NUM_PROC=${PARALLEL_LEVEL:-$(nproc)}
 export BUILD_CC=${BUILD_CC:-"gcc"}
 
-export CONDA_ENV_YML="${MRC_ROOT}/conda/environments/all_cuda-121_arch-x86_64.yaml"
+export CONDA_ENV_YML="${MRC_ROOT}/conda/environments/all_cuda-125_arch-x86_64.yaml"
 
 export CMAKE_BUILD_ALL_FEATURES="-DCMAKE_MESSAGE_CONTEXT_SHOW=ON -DMRC_BUILD_BENCHMARKS=ON -DMRC_BUILD_EXAMPLES=ON -DMRC_BUILD_PYTHON=ON -DMRC_BUILD_TESTS=ON -DMRC_USE_CONDA=ON -DMRC_PYTHON_BUILD_STUBS=ON"
 export CMAKE_BUILD_WITH_CODECOV="-DCMAKE_BUILD_TYPE=Debug -DMRC_ENABLE_CODECOV=ON -DMRC_PYTHON_PERFORM_INSTALL:BOOL=ON -DMRC_PYTHON_INPLACE_BUILD:BOOL=ON"

ci/scripts/run_ci_local.sh

@@ -58,7 +58,7 @@ GIT_BRANCH=$(git branch --show-current)
 GIT_COMMIT=$(git log -n 1 --pretty=format:%H)
 
 BASE_LOCAL_CI_TMP=${BASE_LOCAL_CI_TMP:-${MRC_ROOT}/.tmp/local_ci_tmp}
-CONTAINER_VER=${CONTAINER_VER:-240214}
+CONTAINER_VER=${CONTAINER_VER:-241002}
 CUDA_VER=${CUDA_VER:-12.1}
 DOCKER_EXTRA_ARGS=${DOCKER_EXTRA_ARGS:-""}