Merge branch 'branch-23.07' into mdd_combining-operators

CMakeLists.txt

@@ -36,7 +36,7 @@ option(MRC_USE_CONDA "Enables finding dependencies via conda. All dependencies m
   environment" ON)
 option(MRC_USE_IWYU "Enable running include-what-you-use as part of the build process" OFF)
 
-set(MRC_RAPIDS_VERSION "23.02" CACHE STRING "Which version of RAPIDS to build for. Sets default versions for RAPIDS CMake and RMM.")
+set(MRC_RAPIDS_VERSION "23.06" CACHE STRING "Which version of RAPIDS to build for. Sets default versions for RAPIDS CMake and RMM.")
 
 set(MRC_CACHE_DIR "${CMAKE_SOURCE_DIR}/.cache" CACHE PATH "Directory to contain all CPM and CCache data")
 mark_as_advanced(MRC_CACHE_DIR)

ci/conda/environments/dev_env.yml

@@ -38,17 +38,17 @@ dependencies:
   - gflags=2.2
   - git>=2.35.3 # Needed for wildcards on safe.directory
   - glog=0.6
-  - gmock=1.10
+  - gmock=1.13
   - graphviz=3.0
-  - grpc-cpp=1.48
-  - gtest=1.10
+  - grpc-cpp=1.51
+  - gtest=1.13
   - gxx=11.2 # Specifies which versions of GXX and GCC to use
   - isort
   - jinja2=3.0
   - lcov=1.15
   - libhwloc=2.5
   - libprotobuf=3.21
-  - librmm=23.02
+  - librmm=23.06
   - libtool
   - ninja=1.10
   - nlohmann_json=3.9
@@ -61,9 +61,8 @@ dependencies:
   - pytest-timeout
   - python=3.10
   - scikit-build>=0.17
-  - spdlog=1.8.5
   - sysroot_linux-64=2.17
-  - ucx=1.13
+  - ucx=1.14
   - yapf
 
   # Remove once `mamba repoquery whoneeds cudatoolkit` is empty. For now, we need to specify a version

ci/conda/recipes/libmrc/build.sh

@@ -53,7 +53,7 @@ export CXX=${GXX}
 # Common CMake args
 CMAKE_ARGS="-DBUILD_SHARED_LIBS=ON ${CMAKE_ARGS}"
 CMAKE_ARGS="-DCMAKE_BUILD_TYPE=${MRC_BUILD_TYPE} ${CMAKE_ARGS}"
-CMAKE_ARGS="-DCMAKE_CUDA_ARCHITECTURES=${CMAKE_CUDA_ARCHITECTURES=-"ALL"} ${CMAKE_ARGS}"
+CMAKE_ARGS="-DCMAKE_CUDA_ARCHITECTURES=${CMAKE_CUDA_ARCHITECTURES=-"RAPIDS"} ${CMAKE_ARGS}"
 CMAKE_ARGS="-DCMAKE_INSTALL_LIBDIR=lib ${CMAKE_ARGS}"
 CMAKE_ARGS="-DCMAKE_INSTALL_PREFIX=$PREFIX ${CMAKE_ARGS}"
 CMAKE_ARGS="-DCMAKE_MESSAGE_CONTEXT_SHOW=ON ${CMAKE_ARGS}"

ci/conda/recipes/libmrc/conda_build_config.yaml

@@ -28,30 +28,36 @@ cuda_compiler_version:
 python:
   - 3.8
   - 3.10
+  - 3.10
 
 # Setup the dependencies to build with multiple versions of RAPIDS
-rapids_version: # Keep around compatibility with current version -1
-  - 22.12
+rapids_version: # Keep around compatibility with current version -2
   - 23.02
+  - 23.04
+  - 23.06
 
 # Multiple versions of abseil are required to satisfy the solver for some
 # environments. RAPIDS 22.06 only works with gRPC 1.45 and 22.08 only works with
 # 1.46. For each version of gRPC, support 2 abseil versions. Zip all of the keys
 # together to avoid impossible combinations
 libabseil:
-  - 20220623.0
+  - 20230125.0
+  - 20230125.0
   - 20230125.0
 
 grpc_cpp:
-  - 1.46
+  - 1.51
+  - 1.51
   - 1.51
 
 ucx:
   - 1.13
-  - 1.13
+  - 1.14
+  - 1.14
 
 libprotobuf:
-  - 3.20
+  - 3.21
+  - 3.21
   - 3.21
 
 zip_keys:

ci/conda/recipes/libmrc/meta.yaml

@@ -29,7 +29,7 @@ source:
 build:
   number: {{ GIT_DESCRIBE_NUMBER }}
   script_env:
-    - CMAKE_CUDA_ARCHITECTURES=ALL
+    - CMAKE_CUDA_ARCHITECTURES=RAPIDS
     - MRC_CACHE_DIR
     - PARALLEL_LEVEL
   include_recipe: False
@@ -58,9 +58,9 @@ requirements:
     - flatbuffers 2.0.*
     - gflags
     - glog
-    - gmock 1.10.*
+    - gmock 1.13.*
     - grpc-cpp
-    - gtest 1.10.*
+    - gtest 1.13.*
     - libabseil
     - libhwloc 2.5.*
     - libprotobuf
@@ -70,7 +70,6 @@ requirements:
     - pybind11-stubgen 0.10.5
     - python {{ python }}
     - scikit-build >=0.17
-    - spdlog 1.8.5
     - ucx
     # Need to specify cudatoolkit to get correct version. Remove once all libraries migrate to cuda-toolkit
     - cudatoolkit {{ cuda_version }}.*

ci/conda/recipes/run_conda_build.sh

@@ -53,6 +53,9 @@ export CMAKE_C_COMPILER_LAUNCHER="ccache"
 export CMAKE_CXX_COMPILER_LAUNCHER="ccache"
 export CMAKE_CUDA_COMPILER_LAUNCHER="ccache"
 
+# Avoid confirmation messages during the conda build
+export CONDA_ALWAYS_YES=true
+
 # Holds the arguments in an array to allow for complex json objects
 CONDA_ARGS_ARRAY=()
 
@@ -92,7 +95,7 @@ fi
 # Choose default variants
 if hasArg quick; then
    # For quick build, just do most recent version of rapids
-   CONDA_ARGS_ARRAY+=("--variants" "{rapids_version: 23.02}")
+   CONDA_ARGS_ARRAY+=("--variants" "{rapids_version: 23.06}")
 fi
 
 # And default channels

ci/iwyu/mappings.imp

@@ -27,7 +27,10 @@
 
 # gprc
 { "include": ["@<grpcpp/impl/.*>", "private", "<grpcpp/grpcpp.h>", "public"] },
+{ "include": ["@<grpcpp/.*>", "private", "<grpcpp/grpcpp.h>", "public"] },
 { "include": ["@<grpc/impl/.*>", "private", "<grpcpp/grpcpp.h>", "public"] },
+{ "include": ["@<grpc/.*>", "private", "<grpcpp/grpcpp.h>", "public"] },
+{ "symbol": ["@grpc::.*", "private", "<grpcpp/grpcpp.h>", "public"] },
 
 # nlohmann json
 { "include": ["<nlohmann/detail/iterators/iter_impl.hpp>", "private", "<nlohmann/json.hpp>", "public"] },

ci/scripts/github/common.sh

@@ -80,7 +80,7 @@ function update_conda_env() {
 
     # Make sure we have the conda-merge package installed
     if [[ -z "$(conda list | grep conda-merge)" ]]; then
-        rapids-mamba-retry install -n mrc -c conda-forge "conda-merge>=0.2"
+        rapids-mamba-retry install -q -n mrc -c conda-forge "conda-merge>=0.2"
     fi
 
     # Create a temp directory which we store the combined environment file in
@@ -91,7 +91,7 @@ function update_conda_env() {
     conda run -n mrc --live-stream conda-merge ${CONDA_ENV_YML} ${CONDA_CLANG_ENV_YML} ${CONDA_CI_ENV_YML} > ${condatmpdir}/merged_env.yml
 
     # Update the conda env with prune remove excess packages (in case one was removed from the env)
-    rapids-mamba-retry env update -n mrc -q --prune --file ${condatmpdir}/merged_env.yml
+    rapids-mamba-retry env update -n mrc --prune --file ${condatmpdir}/merged_env.yml
 
     # Delete the temp directory
     rm -rf ${condatmpdir}

cpp/mrc/CMakeLists.txt

@@ -21,36 +21,34 @@ add_library(libmrc
   src/internal/codable/codable_storage.cpp
   src/internal/codable/decodable_storage_view.cpp
   src/internal/codable/storage_view.cpp
-  src/internal/control_plane/client/connections_manager.cpp
   src/internal/control_plane/client.cpp
+  src/internal/control_plane/client/connections_manager.cpp
   src/internal/control_plane/client/instance.cpp
   src/internal/control_plane/client/state_manager.cpp
   src/internal/control_plane/client/subscription_service.cpp
-  src/internal/control_plane/resources.cpp
-  src/internal/control_plane/server/connection_manager.cpp
+  src/internal/control_plane/control_plane_resources.cpp
   src/internal/control_plane/server.cpp
+  src/internal/control_plane/server/connection_manager.cpp
   src/internal/control_plane/server/subscription_manager.cpp
   src/internal/control_plane/server/tagged_issuer.cpp
   src/internal/data_plane/callbacks.cpp
   src/internal/data_plane/client.cpp
+  src/internal/data_plane/data_plane_resources.cpp
   src/internal/data_plane/request.cpp
-  src/internal/data_plane/resources.cpp
   src/internal/data_plane/server.cpp
-  src/internal/executor/executor.cpp
-  src/internal/executor/iexecutor.cpp
+  src/internal/executor/executor_definition.cpp
   src/internal/grpc/progress_engine.cpp
   src/internal/grpc/server.cpp
   src/internal/memory/device_resources.cpp
   src/internal/memory/host_resources.cpp
   src/internal/memory/transient_pool.cpp
-  src/internal/network/resources.cpp
+  src/internal/network/network_resources.cpp
   src/internal/pipeline/controller.cpp
-  src/internal/pipeline/instance.cpp
-  src/internal/pipeline/ipipeline.cpp
   src/internal/pipeline/manager.cpp
-  src/internal/pipeline/pipeline.cpp
+  src/internal/pipeline/pipeline_definition.cpp
+  src/internal/pipeline/pipeline_instance.cpp
+  src/internal/pipeline/pipeline_resources.cpp
   src/internal/pipeline/port_graph.cpp
-  src/internal/pipeline/resources.cpp
   src/internal/pubsub/publisher_round_robin.cpp
   src/internal/pubsub/publisher_service.cpp
   src/internal/pubsub/subscriber_service.cpp
@@ -61,21 +59,19 @@ add_library(libmrc
   src/internal/resources/manager.cpp
   src/internal/resources/partition_resources_base.cpp
   src/internal/resources/partition_resources.cpp
-  src/internal/runnable/engine.cpp
   src/internal/runnable/engine_factory.cpp
+  src/internal/runnable/engine.cpp
   src/internal/runnable/engines.cpp
   src/internal/runnable/fiber_engine.cpp
   src/internal/runnable/fiber_engines.cpp
-  src/internal/runnable/resources.cpp
+  src/internal/runnable/runnable_resources.cpp
   src/internal/runnable/thread_engine.cpp
   src/internal/runnable/thread_engines.cpp
   src/internal/runtime/partition.cpp
   src/internal/runtime/runtime.cpp
-  src/internal/segment/builder.cpp
-  src/internal/segment/definition.cpp
-  src/internal/segment/ibuilder.cpp
-  src/internal/segment/idefinition.cpp
-  src/internal/segment/instance.cpp
+  src/internal/segment/builder_definition.cpp
+  src/internal/segment/segment_definition.cpp
+  src/internal/segment/segment_instance.cpp
   src/internal/service.cpp
   src/internal/system/device_info.cpp
   src/internal/system/device_partition.cpp
@@ -84,39 +80,36 @@ add_library(libmrc
   src/internal/system/fiber_pool.cpp
   src/internal/system/fiber_task_queue.cpp
   src/internal/system/gpu_info.cpp
-  src/internal/system/host_partition.cpp
   src/internal/system/host_partition_provider.cpp
-  src/internal/system/iresources.cpp
-  src/internal/system/isystem.cpp
-  src/internal/system/partition.cpp
+  src/internal/system/host_partition.cpp
   src/internal/system/partition_provider.cpp
+  src/internal/system/partition.cpp
   src/internal/system/partitions.cpp
-  src/internal/system/resources.cpp
-  src/internal/system/system.cpp
   src/internal/system/system_provider.cpp
-  src/internal/system/thread.cpp
+  src/internal/system/system.cpp
   src/internal/system/thread_pool.cpp
+  src/internal/system/thread.cpp
+  src/internal/system/threading_resources.cpp
   src/internal/system/topology.cpp
   src/internal/ucx/context.cpp
   src/internal/ucx/endpoint.cpp
   src/internal/ucx/memory_block.cpp
   src/internal/ucx/receive_manager.cpp
-  src/internal/ucx/resources.cpp
+  src/internal/ucx/ucx_resources.cpp
   src/internal/ucx/worker.cpp
   src/internal/utils/collision_detector.cpp
   src/internal/utils/exception_guard.cpp
   src/internal/utils/parse_config.cpp
   src/internal/utils/parse_ints.cpp
   src/internal/utils/shared_resource_bit_map.cpp
-  src/public/benchmarking/tracer.cpp
   src/public/benchmarking/trace_statistics.cpp
+  src/public/benchmarking/tracer.cpp
   src/public/benchmarking/util.cpp
   src/public/channel/channel.cpp
   src/public/codable/encoded_object.cpp
   src/public/codable/memory.cpp
   src/public/core/addresses.cpp
   src/public/core/bitmap.cpp
-  src/public/core/executor.cpp
   src/public/core/fiber_pool.cpp
   src/public/core/logging.cpp
   src/public/core/thread.cpp
@@ -145,15 +138,16 @@ add_library(libmrc
   src/public/options/resources.cpp
   src/public/options/services.cpp
   src/public/options/topology.cpp
+  src/public/pipeline/executor.cpp
   src/public/pipeline/pipeline.cpp
+  src/public/pipeline/segment.cpp
+  src/public/pipeline/system.cpp
   src/public/runnable/context.cpp
   src/public/runnable/launcher.cpp
   src/public/runnable/runnable.cpp
   src/public/runnable/runner.cpp
   src/public/runnable/types.cpp
   src/public/runtime/remote_descriptor.cpp
-  src/public/segment/builder.cpp
-  src/public/segment/definition.cpp
   src/public/utils/bytes_to_string.cpp
   src/public/utils/thread_utils.cpp
   src/public/utils/type_utils.cpp

cpp/mrc/benchmarks/bench_segment.cpp

@@ -18,11 +18,10 @@
 #include "mrc/benchmarking/segment_watcher.hpp"
 #include "mrc/benchmarking/tracer.hpp"
 #include "mrc/benchmarking/util.hpp"
-#include "mrc/core/executor.hpp"
-#include "mrc/engine/pipeline/ipipeline.hpp"
 #include "mrc/node/rx_node.hpp"
 #include "mrc/node/rx_sink.hpp"
 #include "mrc/node/rx_source.hpp"
+#include "mrc/pipeline/executor.hpp"
 #include "mrc/pipeline/pipeline.hpp"
 #include "mrc/segment/builder.hpp"  // IWYU pragma: keep
 #include "mrc/segment/object.hpp"   // IWYU pragma: keep
@@ -190,7 +189,7 @@ class SimpleEmitReceiveFixture : public benchmark::Fixture
     void SetUp(const ::benchmark::State& state) override
     {
         TimeUtil::estimate_steady_clock_delay();
-        auto init = [this](segment::Builder& segment) {
+        auto init = [this](segment::IBuilder& segment) {
             std::string src_name  = "nsrc";
             std::string int_name  = "n1";
             std::string sink_name = "nsink";
@@ -215,7 +214,7 @@ class SimpleEmitReceiveFixture : public benchmark::Fixture
             segment.make_edge(internal, sink);
         };
 
-        auto pipeline = pipeline::make_pipeline();
+        auto pipeline = mrc::make_pipeline();
         auto segment  = pipeline->make_segment("bench_segment", init);
 
         std::shared_ptr<Executor> executor = std::make_shared<Executor>();
@@ -250,7 +249,7 @@ class LongEmitReceiveFixture : public benchmark::Fixture
     void SetUp(const ::benchmark::State& state) override
     {
         TimeUtil::estimate_steady_clock_delay();
-        auto init = [this](segment::Builder& segment) {
+        auto init = [this](segment::IBuilder& segment) {
             std::string src_name  = "nsrc";
             std::string sink_name = "nsink";
 
@@ -284,7 +283,7 @@ class LongEmitReceiveFixture : public benchmark::Fixture
             segment.make_edge(last_node, sink);
         };
 
-        auto pipeline = pipeline::make_pipeline();
+        auto pipeline = mrc::make_pipeline();
         auto segment  = pipeline->make_segment("bench_segment", init);
 
         std::shared_ptr<Executor> executor = std::make_shared<Executor>();