Merge branch 'branch-23.11' into branch-24.03-merge-23.11

nv-morpheus · Dec 6, 2023 · 960b8d1 · 960b8d1
2 parents b152175 + de48cb1
commit 960b8d1
Show file tree

Hide file tree

Showing 31 changed files with 45 additions and 63 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,37 @@
+# MRC 23.11.00 (30 Nov 2023)
+
+## 🐛 Bug Fixes
+
+- Use a traditional semaphore in AsyncioRunnable ([#412](https://github.com/nv-morpheus/MRC/pull/412)) [@cwharris](https://github.com/cwharris)
+- Fix libhwloc &amp; stubgen versions to match dev yaml ([#405](https://github.com/nv-morpheus/MRC/pull/405)) [@dagardner-nv](https://github.com/dagardner-nv)
+- Update boost versions to match version used in dev env ([#404](https://github.com/nv-morpheus/MRC/pull/404)) [@dagardner-nv](https://github.com/dagardner-nv)
+- Fix EdgeHolder from incorrectly reporting an active connection ([#402](https://github.com/nv-morpheus/MRC/pull/402)) [@dagardner-nv](https://github.com/dagardner-nv)
+- Safe handling of control plane promises &amp; fix CI ([#391](https://github.com/nv-morpheus/MRC/pull/391)) [@dagardner-nv](https://github.com/dagardner-nv)
+- Revert boost upgrade, and update clang to v16 ([#382](https://github.com/nv-morpheus/MRC/pull/382)) [@dagardner-nv](https://github.com/dagardner-nv)
+- Fixing an issue with `update-versions.sh` which always blocked CI ([#377](https://github.com/nv-morpheus/MRC/pull/377)) [@mdemoret-nv](https://github.com/mdemoret-nv)
+- Add test for  gc being invoked in a thread finalizer ([#365](https://github.com/nv-morpheus/MRC/pull/365)) [@dagardner-nv](https://github.com/dagardner-nv)
+- Adopt patched pybind11 ([#364](https://github.com/nv-morpheus/MRC/pull/364)) [@dagardner-nv](https://github.com/dagardner-nv)
+
+## 📖 Documentation
+
+- Add missing flags to docker command to mount the working dir and set -cap-add=sys_nice ([#383](https://github.com/nv-morpheus/MRC/pull/383)) [@dagardner-nv](https://github.com/dagardner-nv)
+- Make Quick Start Guide not use `make_node_full` ([#376](https://github.com/nv-morpheus/MRC/pull/376)) [@cwharris](https://github.com/cwharris)
+
+## 🚀 New Features
+
+- Add AsyncioRunnable ([#411](https://github.com/nv-morpheus/MRC/pull/411)) [@cwharris](https://github.com/cwharris)
+- Adding more coroutine components to support async generators and task containers ([#408](https://github.com/nv-morpheus/MRC/pull/408)) [@mdemoret-nv](https://github.com/mdemoret-nv)
+- Update ObservableProxy::pipe to support any number of operators ([#387](https://github.com/nv-morpheus/MRC/pull/387)) [@cwharris](https://github.com/cwharris)
+- Updates for MRC/Morpheus to build in the same RAPIDS devcontainer environment ([#375](https://github.com/nv-morpheus/MRC/pull/375)) [@cwharris](https://github.com/cwharris)
+
+## 🛠️ Improvements
+
+- Move Pycoro from Morpheus to MRC ([#409](https://github.com/nv-morpheus/MRC/pull/409)) [@cwharris](https://github.com/cwharris)
+- update rapidsai/ci to rapidsai/ci-conda ([#396](https://github.com/nv-morpheus/MRC/pull/396)) [@AyodeAwe](https://github.com/AyodeAwe)
+- Add local CI scripts &amp; rebase docker image ([#394](https://github.com/nv-morpheus/MRC/pull/394)) [@dagardner-nv](https://github.com/dagardner-nv)
+- Use `copy-pr-bot` ([#369](https://github.com/nv-morpheus/MRC/pull/369)) [@ajschmidt8](https://github.com/ajschmidt8)
+- Update Versions for v23.11.00 ([#357](https://github.com/nv-morpheus/MRC/pull/357)) [@mdemoret-nv](https://github.com/mdemoret-nv)
+
 # MRC 23.07.00 (19 Jul 2023)
 
 ## 🚨 Breaking Changes

diff --git a/ci/conda/environments/dev_env.yml b/ci/conda/environments/dev_env.yml
@@ -27,7 +27,7 @@ dependencies:
   - benchmark=1.6.0
   - boost-cpp=1.82
   - ccache
-  - cmake=3.24
+  - cmake=3.25
   - cuda-toolkit # Version comes from the channel above
   - cxx-compiler # Sets up the distro versions of our compilers
   - doxygen=1.9.2

diff --git a/external/utilities b/external/utilities
diff --git a/python/mrc/_pymrc/include/pymrc/asyncio_runnable.hpp b/python/mrc/_pymrc/include/pymrc/asyncio_runnable.hpp
@@ -212,7 +212,7 @@ class AsyncioRunnable : public AsyncSink<InputT>,
     /**
      * @brief A semaphore used to control the number of outstanding operations. Acquire one before
      * beginning a task, and release it when finished.
-    */
+     */
     std::counting_semaphore<8> m_task_tickets{8};
 };
 

diff --git a/python/mrc/_pymrc/include/pymrc/coro.hpp b/python/mrc/_pymrc/include/pymrc/coro.hpp
@@ -31,7 +31,9 @@
 #include <coroutine>
 #include <exception>
 #include <memory>
-#include <ostream>
+#include <sstream>    // for operator<<, basic_ostringstream
+#include <stdexcept>  // for runtime_error
+#include <string>     // for string
 #include <utility>
 
 // Dont directly include python headers

diff --git a/python/mrc/_pymrc/include/pymrc/utilities/function_wrappers.hpp b/python/mrc/_pymrc/include/pymrc/utilities/function_wrappers.hpp
@@ -27,7 +27,6 @@
 #include <pybind11/pybind11.h>
 #include <pybind11/pytypes.h>
 
-#include <array>
 #include <exception>
 #include <functional>
 #include <sstream>

diff --git a/python/mrc/_pymrc/src/executor.cpp b/python/mrc/_pymrc/src/executor.cpp
@@ -25,7 +25,6 @@
 #include "mrc/types.hpp"
 
 #include <boost/fiber/future/async.hpp>
-#include <boost/fiber/future/future.hpp>
 #include <boost/fiber/future/future_status.hpp>
 #include <glog/logging.h>
 #include <pybind11/cast.h>

diff --git a/python/mrc/_pymrc/src/module_registry.cpp b/python/mrc/_pymrc/src/module_registry.cpp
@@ -28,7 +28,6 @@
 #include <pybind11/pybind11.h>
 #include <pybind11/pytypes.h>
 
-#include <array>
 #include <memory>
 #include <ostream>
 #include <utility>

diff --git a/python/mrc/_pymrc/src/module_wrappers/pickle.cpp b/python/mrc/_pymrc/src/module_wrappers/pickle.cpp
@@ -24,7 +24,6 @@
 #include <pybind11/pybind11.h>
 #include <pybind11/pytypes.h>
 
-#include <array>
 #include <memory>
 #include <ostream>
 

diff --git a/python/mrc/_pymrc/src/module_wrappers/shared_memory.cpp b/python/mrc/_pymrc/src/module_wrappers/shared_memory.cpp
@@ -20,10 +20,9 @@
 #include "pymrc/utilities/object_cache.hpp"
 
 #include <pybind11/cast.h>
-#include <pybind11/pybind11.h>
+#include <pybind11/pybind11.h>  // IWYU pragma: keep
 #include <pybind11/pytypes.h>
 
-#include <array>
 #include <cstddef>
 #include <stdexcept>
 #include <string>

diff --git a/python/mrc/_pymrc/src/segment.cpp b/python/mrc/_pymrc/src/segment.cpp
@@ -28,12 +28,9 @@
 #include "mrc/channel/status.hpp"
 #include "mrc/edge/edge_builder.hpp"
 #include "mrc/node/port_registry.hpp"
-#include "mrc/node/rx_sink_base.hpp"
-#include "mrc/node/rx_source_base.hpp"
 #include "mrc/runnable/context.hpp"
 #include "mrc/segment/builder.hpp"
 #include "mrc/segment/object.hpp"
-#include "mrc/types.hpp"
 
 #include <glog/logging.h>
 #include <pybind11/cast.h>
@@ -44,15 +41,13 @@
 #include <exception>
 #include <fstream>
 #include <functional>
-#include <future>
 #include <iterator>
 #include <map>
 #include <stdexcept>
 #include <string>
 #include <type_traits>
 #include <typeindex>
 #include <utility>
-#include <vector>
 
 // IWYU thinks we need array for py::print
 // IWYU pragma: no_include <array>

diff --git a/python/mrc/_pymrc/src/subscriber.cpp b/python/mrc/_pymrc/src/subscriber.cpp
@@ -28,7 +28,6 @@
 #include <pybind11/pytypes.h>
 #include <rxcpp/rx.hpp>
 
-#include <array>
 #include <exception>
 #include <functional>
 #include <stdexcept>

diff --git a/python/mrc/_pymrc/src/utilities/object_cache.cpp b/python/mrc/_pymrc/src/utilities/object_cache.cpp
@@ -24,7 +24,6 @@
 #include <pybind11/pytypes.h>
 #include <pylifecycle.h>
 
-#include <array>
 #include <mutex>
 #include <ostream>
 #include <stdexcept>

diff --git a/python/mrc/_pymrc/src/watchers.cpp b/python/mrc/_pymrc/src/watchers.cpp
@@ -24,8 +24,8 @@
 #include "mrc/benchmarking/tracer.hpp"
 #include "mrc/node/rx_node.hpp"
 #include "mrc/node/rx_sink.hpp"
-#include "mrc/node/rx_source.hpp"
 #include "mrc/segment/builder.hpp"
+#include "mrc/segment/object.hpp"
 
 #include <nlohmann/json.hpp>
 #include <pybind11/gil.h>
@@ -34,11 +34,9 @@
 
 #include <cstddef>
 #include <functional>
-#include <map>
 #include <memory>
 #include <string>
 #include <utility>
-#include <vector>
 
 namespace mrc::pymrc {
 

diff --git a/python/mrc/_pymrc/tests/test_asyncio_runnable.cpp b/python/mrc/_pymrc/tests/test_asyncio_runnable.cpp
@@ -80,7 +80,7 @@ class __attribute__((visibility("default"))) TestAsyncioRunnable : public ::test
 
 std::unique_ptr<pybind11::scoped_interpreter> TestAsyncioRunnable::m_interpreter;
 
-class PythonCallbackAsyncioRunnable : public pymrc::AsyncioRunnable<int, int>
+class __attribute__((visibility("default"))) PythonCallbackAsyncioRunnable : public pymrc::AsyncioRunnable<int, int>
 {
   public:
     PythonCallbackAsyncioRunnable(pymrc::PyObjectHolder operation) : m_operation(std::move(operation)) {}

diff --git a/python/mrc/_pymrc/tests/test_executor.cpp b/python/mrc/_pymrc/tests/test_executor.cpp
@@ -33,11 +33,9 @@
 #include <rxcpp/rx.hpp>
 
 #include <atomic>
-#include <map>
 #include <memory>
 #include <string>
 #include <utility>
-#include <vector>
 
 namespace py    = pybind11;
 namespace pymrc = mrc::pymrc;

diff --git a/python/mrc/_pymrc/tests/test_pipeline.cpp b/python/mrc/_pymrc/tests/test_pipeline.cpp
@@ -31,9 +31,7 @@
 #include "mrc/options/topology.hpp"
 #include "mrc/segment/builder.hpp"
 #include "mrc/segment/object.hpp"
-#include "mrc/types.hpp"
 
-#include <boost/fiber/future/future.hpp>
 #include <glog/logging.h>
 #include <gtest/gtest.h>
 #include <pybind11/cast.h>
@@ -46,7 +44,6 @@
 #include <cstddef>
 #include <functional>
 #include <iostream>
-#include <map>
 #include <memory>
 #include <stdexcept>
 #include <string>

diff --git a/python/mrc/_pymrc/tests/test_serializers.cpp b/python/mrc/_pymrc/tests/test_serializers.cpp
@@ -28,7 +28,6 @@
 #include <pybind11/pytypes.h>
 #include <pybind11/stl.h>  // IWYU pragma: keep
 
-#include <array>
 #include <ostream>
 #include <stdexcept>
 #include <string>

diff --git a/python/mrc/_pymrc/tests/test_utils.cpp b/python/mrc/_pymrc/tests/test_utils.cpp
@@ -34,7 +34,6 @@
 #include <climits>
 #include <map>
 #include <memory>
-#include <stdexcept>
 #include <string>
 #include <utility>
 #include <vector>

diff --git a/python/mrc/benchmarking/watchers.cpp b/python/mrc/benchmarking/watchers.cpp
@@ -26,11 +26,9 @@
 #include <pybind11/gil.h>  // IWYU pragma: keep
 #include <pybind11/pybind11.h>
 
-#include <array>
 #include <cstddef>
 #include <functional>
 #include <memory>
-#include <vector>
 
 namespace mrc::pymrc {
 namespace py = pybind11;

diff --git a/python/mrc/core/common.cpp b/python/mrc/core/common.cpp
@@ -18,21 +18,15 @@
 #include "pymrc/port_builders.hpp"
 #include "pymrc/types.hpp"
 
-#include "mrc/node/rx_sink_base.hpp"
-#include "mrc/node/rx_source_base.hpp"
-#include "mrc/types.hpp"
 #include "mrc/utils/string_utils.hpp"
 #include "mrc/version.hpp"
 
-#include <boost/fiber/future/future.hpp>
 #include <pybind11/pybind11.h>
 #include <pybind11/pytypes.h>
 #include <rxcpp/rx.hpp>
 
-#include <map>
 #include <memory>
 #include <sstream>
-#include <vector>
 
 namespace mrc::pymrc {
 

diff --git a/python/mrc/core/coro.cpp b/python/mrc/core/coro.cpp
@@ -25,7 +25,6 @@
 
 #include <coroutine>
 #include <memory>
-#include <ostream>
 #include <string>
 #include <vector>
 

diff --git a/python/mrc/core/operators.cpp b/python/mrc/core/operators.cpp
@@ -28,7 +28,6 @@
 #include <pybind11/pytypes.h>
 #include <pybind11/stl.h>  // IWYU pragma: keep
 
-#include <array>
 #include <sstream>
 
 namespace mrc::pymrc {

diff --git a/python/mrc/core/pipeline.cpp b/python/mrc/core/pipeline.cpp
@@ -27,7 +27,6 @@
 #include <pybind11/pybind11.h>
 #include <pybind11/stl.h>  // IWYU pragma: keep
 
-#include <array>
 #include <sstream>
 
 namespace mrc::pymrc {

diff --git a/python/mrc/core/segment.cpp b/python/mrc/core/segment.cpp
@@ -38,12 +38,9 @@
 #include <pybind11/pybind11.h>
 #include <pybind11/pytypes.h>
 
-#include <array>
-#include <cstddef>
 #include <cstdint>
 #include <memory>
 #include <sstream>
-#include <vector>
 
 namespace mrc::pymrc {
 

diff --git a/python/mrc/core/segment/module_definitions/mirror_tap_orchestrator.cpp b/python/mrc/core/segment/module_definitions/mirror_tap_orchestrator.cpp
@@ -26,18 +26,14 @@
 #include "mrc/experimental/modules/stream_buffer/stream_buffer_module.hpp"
 #include "mrc/modules/module_registry.hpp"
 #include "mrc/modules/module_registry_util.hpp"
-#include "mrc/node/operators/broadcast.hpp"
-#include "mrc/node/rx_sink.hpp"
-#include "mrc/node/rx_source.hpp"
 #include "mrc/version.hpp"
 
 #include <pybind11/cast.h>
 #include <pybind11/functional.h>  // IWYU pragma: keep
 #include <pybind11/pybind11.h>
 #include <pybind11/pytypes.h>
+#include <rxcpp/rx.hpp>
 
-#include <array>
-#include <map>
 #include <memory>
 #include <string>
 #include <vector>

diff --git a/python/mrc/core/segment/module_definitions/segment_module_registry.cpp b/python/mrc/core/segment/module_definitions/segment_module_registry.cpp
@@ -25,12 +25,9 @@
 #include <pybind11/cast.h>
 #include <pybind11/functional.h>  // IWYU pragma: keep
 #include <pybind11/pybind11.h>
-#include <pybind11/pytypes.h>
 #include <pybind11/stl.h>  // IWYU pragma: keep
 
-#include <array>
 #include <functional>
-#include <map>
 #include <string>
 #include <vector>
 

diff --git a/python/mrc/core/segment/module_definitions/segment_modules.cpp b/python/mrc/core/segment/module_definitions/segment_modules.cpp
@@ -25,9 +25,7 @@
 #include <pybind11/cast.h>
 #include <pybind11/pybind11.h>
 
-#include <map>
 #include <memory>
-#include <vector>
 
 namespace mrc::pymrc {
 

diff --git a/python/mrc/core/subscriber.cpp b/python/mrc/core/subscriber.cpp
@@ -27,8 +27,8 @@
 #include <pybind11/functional.h>  // IWYU pragma: keep
 #include <pybind11/gil.h>         // IWYU pragma: keep(for call_guard)
 #include <pybind11/pybind11.h>
+#include <rxcpp/rx.hpp>
 
-#include <array>
 #include <memory>
 #include <sstream>
 

diff --git a/python/mrc/tests/sample_modules.cpp b/python/mrc/tests/sample_modules.cpp
@@ -20,15 +20,12 @@
 #include "pymrc/utils.hpp"
 
 #include "mrc/modules/module_registry_util.hpp"
-#include "mrc/node/rx_source.hpp"
 #include "mrc/utils/string_utils.hpp"
 #include "mrc/version.hpp"
 
 #include <pybind11/pybind11.h>
 #include <pybind11/pytypes.h>
 
-#include <map>
-#include <memory>
 #include <sstream>
 #include <vector>