diff --git a/CMakeLists.txt b/CMakeLists.txt index 80e7220..dddf8c4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -183,11 +183,10 @@ if(${CMAKE_PROJECT_NAME} STREQUAL ${PROJECT_NAME}) if (ENABLE_EXAMPLES) message(STATUS "Building example programs. Turn it off via ENABLE_EXAMPLES=OFF") - add_subdirectory(examples/lockless_example) + add_subdirectory(examples/lockless_examples) add_subdirectory(examples/logging_example) add_subdirectory(examples/message_passing_example) add_subdirectory(examples/mutex_example) - add_subdirectory(examples/signal_handling_example) add_subdirectory(examples/simple_deadline_example) add_subdirectory(examples/simple_example) add_subdirectory(examples/random_example) diff --git a/README.md b/README.md index ebe6840..c45aeee 100644 --- a/README.md +++ b/README.md @@ -29,22 +29,22 @@ writing a real-time Linux application. Some key features are: Examples -------- -See each example's README for more details on what they do. - * [`simple_example`](examples/simple_example/): The most basic example showing - a single real-time looping thread. -* [`signal_handling_example`](examples/signal_handling_example/): Same as - `simple_example`, except the program respond to SIGTERM and SIGINT and quit - upon receiving the signal. -* [`logging_example`](examples/logging_example/): Demonstrates setting up custom - logging configuration via `cactus_rt::App`. + a single real-time looping thread running at 1000 Hz. +* [`tracing_example`](examples/tracing_example/): This demonstrates how to use + the real-time-safe tracing system built into cactus-rt. This is probably a + good thing to undrestand immediately after the above example. * [`mutex_example`](examples/mutex_example/): Demonstrates the usage of priority-inheritence mutex (`cactus_rt::mutex`) to pass data between real-time - and non-real-time threads. + and non-real-time threads via the implementation of a mutex-based double + buffer. +* [`logging_example`](examples/logging_example/): Demonstrates setting up custom + logging configuration via `cactus_rt::App`. + * [`simple_deadline_example`](examples/simple_deadline_example/): Same as `simple_example`, except it uses `SCHED_DEADLINE` as opposed to `SCHED_FIFO`. This is for a more advanced use case. -* [`tracing_example`](examples/tracing_example/): Shows how to dynamically start and stop tracing, as well as trace custom application functions. + * [`tracing_example_no_rt`](examples/tracing_example_no_rt/): Shows how to using the tracing library in `cactus_rt` without using `cactus_rt::App`. diff --git a/examples/lockless_example/CMakeLists.txt b/examples/lockless_example/CMakeLists.txt deleted file mode 100644 index 90eef58..0000000 --- a/examples/lockless_example/CMakeLists.txt +++ /dev/null @@ -1,10 +0,0 @@ -add_executable(rt_lockless_example - main.cc -) - -target_link_libraries(rt_lockless_example - PRIVATE - cactus_rt -) - -setup_cactus_rt_target_options(rt_lockless_example) diff --git a/examples/lockless_examples/CMakeLists.txt b/examples/lockless_examples/CMakeLists.txt new file mode 100644 index 0000000..3d26d69 --- /dev/null +++ b/examples/lockless_examples/CMakeLists.txt @@ -0,0 +1,10 @@ +add_executable(rt_lockless_realtime_read_example + realtime_read.cc +) + +target_link_libraries(rt_lockless_realtime_read_example + PRIVATE + cactus_rt +) + +setup_cactus_rt_target_options(rt_lockless_realtime_read_example) diff --git a/examples/lockless_examples/README.md b/examples/lockless_examples/README.md new file mode 100644 index 0000000..d6a9438 --- /dev/null +++ b/examples/lockless_examples/README.md @@ -0,0 +1,3 @@ +Lockless examples +================= + diff --git a/examples/lockless_example/main.cc b/examples/lockless_examples/realtime_read.cc similarity index 100% rename from examples/lockless_example/main.cc rename to examples/lockless_examples/realtime_read.cc diff --git a/examples/mutex_example/README.md b/examples/mutex_example/README.md index 6a675b1..d77cd7e 100644 --- a/examples/mutex_example/README.md +++ b/examples/mutex_example/README.md @@ -4,14 +4,17 @@ This program shows the usage of sharing data between an RT and a non-RT thread via the `cactus_rt::mutex`, which is a priority-inheritence mutex compatible with the [`Lockable`](https://en.cppreference.com/w/cpp/named_req/Lockable) -interface. +interface. This means you can use this `mutex` just like you would a normal +mutex from STL and expect that priority inheritance is enabled on it. -Specifically, this example implements a very naive double buffer. This data -structure has 2 data slots guarded by the priority-inheriting mutex. The RT +In this example, we use the `cactus_rt::mutex` to implement a very naive double +buffer. It has 2 data slots guarded by the priority-inheriting mutex. The RT thread writes to the double buffer at 1 kHz and the non-RT thread reads it every half second. Once it is read, it is logged via the `cactus_rt` logging capability. +_Note: a lockless version of this double buffer is implemented by the cactus-rt framework under `cactus_rt::experimental::lockless::spsc::AtomicWritableValue` which doesn't require a lock. That serves as an alternative to this code without the usage of a mutex._ + To run: ```bash diff --git a/examples/mutex_example/double_buffer.h b/examples/mutex_example/double_buffer.h index 5e119b2..160d7e2 100644 --- a/examples/mutex_example/double_buffer.h +++ b/examples/mutex_example/double_buffer.h @@ -3,7 +3,6 @@ #include -#include #include /** @@ -12,7 +11,11 @@ * Also only uses a single mutex so reads, writes, and swaps contend on a * single lock, which is no good for performance. * - * Realistically, you would implement this in a lock-free manner, like this: + * cactus-rt already has something for this use case (real-time thread writes + * and non-real-time thread reads) implemented via the + * cactus_rt::experimental::lockless::spsc::RealtimeWritableValue. + * + * There could also be alternative implementations like: * https://stackoverflow.com/questions/23666069/single-producer-single-consumer-data-structure-with-double-buffer-in-c */ template diff --git a/examples/mutex_example/main.cc b/examples/mutex_example/main.cc index c2c5e96..31945f2 100644 --- a/examples/mutex_example/main.cc +++ b/examples/mutex_example/main.cc @@ -10,6 +10,9 @@ using cactus_rt::App; using cactus_rt::CyclicThread; using cactus_rt::Thread; +// This is the data structure we are passing between the RT and non-RT thread. +// It is big enough such that it cannot be atomically changed with a single +// instruction to necessitate a mutex. struct Data { double v1 = 0.0; double v2 = 0.0; @@ -21,8 +24,8 @@ class RTThread : public CyclicThread { NaiveDoubleBuffer& buf_; public: - explicit RTThread(const char* name, cactus_rt::CyclicThreadConfig config, NaiveDoubleBuffer& buf) - : CyclicThread(name, config), + explicit RTThread(NaiveDoubleBuffer& buf) + : CyclicThread("RTThread", CreateConfig()), buf_(buf) {} protected: @@ -40,14 +43,24 @@ class RTThread : public CyclicThread { return LoopControl::Continue; } + + private: + static cactus_rt::CyclicThreadConfig CreateConfig() { + cactus_rt::CyclicThreadConfig thread_config; + thread_config.period_ns = 1'000'000; + thread_config.SetFifoScheduler(80); + + return thread_config; + } }; class NonRTThread : public Thread { NaiveDoubleBuffer& buf_; public: - explicit NonRTThread(const char* name, cactus_rt::CyclicThreadConfig config, NaiveDoubleBuffer& buf) - : Thread(name, config), buf_(buf) {} + explicit NonRTThread(NaiveDoubleBuffer& buf) + : Thread("NonRTThread", CreateConfig()), + buf_(buf) {} protected: void Run() final { @@ -58,32 +71,33 @@ class NonRTThread : public Thread { std::this_thread::sleep_for(std::chrono::milliseconds(500)); } } + + private: + static cactus_rt::ThreadConfig CreateConfig() { + cactus_rt::CyclicThreadConfig rt_thread_config; + rt_thread_config.SetOtherScheduler(0 /* niceness */); + return rt_thread_config; + } }; +// Trivial demonstration of the double buffer. void TrivialDemo() { - // Trivial demonstration that the double buffer does work.. NaiveDoubleBuffer buf; buf.Write(2); auto a = buf.SwapAndRead(); std::cout << "a is " << a << std::endl; } +// The actual application running. void ThreadedDemo() { - cactus_rt::CyclicThreadConfig rt_thread_config; - rt_thread_config.period_ns = 1'000'000; - rt_thread_config.SetFifoScheduler(80 /* priority */); - - cactus_rt::CyclicThreadConfig non_rt_thread_config; - non_rt_thread_config.SetOtherScheduler(0 /* niceness */); - // The double buffer is shared between the two threads, so we pass a reference // into the thread and maintain the object lifetime to this function. NaiveDoubleBuffer buf; App app; - auto rt_thread = app.CreateThread("RTThread", rt_thread_config, buf); - auto non_rt_thread = app.CreateThread("NonRTThread", non_rt_thread_config, buf); + auto rt_thread = app.CreateThread(buf); + auto non_rt_thread = app.CreateThread(buf); constexpr unsigned int time = 10; app.Start(); diff --git a/examples/signal_handling_example/CMakeLists.txt b/examples/signal_handling_example/CMakeLists.txt deleted file mode 100644 index 89a77a9..0000000 --- a/examples/signal_handling_example/CMakeLists.txt +++ /dev/null @@ -1,10 +0,0 @@ -add_executable(rt_signal_handling_example - main.cc -) - -target_link_libraries(rt_signal_handling_example - PRIVATE - cactus_rt -) - -setup_cactus_rt_target_options(rt_signal_handling_example) diff --git a/examples/signal_handling_example/README.md b/examples/signal_handling_example/README.md deleted file mode 100644 index 3ad122f..0000000 --- a/examples/signal_handling_example/README.md +++ /dev/null @@ -1,4 +0,0 @@ -`simple_example` -================ - -The simplest real-time example to get started with. diff --git a/examples/signal_handling_example/main.cc b/examples/signal_handling_example/main.cc deleted file mode 100644 index ed41d6c..0000000 --- a/examples/signal_handling_example/main.cc +++ /dev/null @@ -1,57 +0,0 @@ -#include - -#include - -using cactus_rt::App; -using cactus_rt::CyclicThread; - -/** - * This is a no-op thread that does nothing at 1 kHz. - */ -class ExampleRTThread : public CyclicThread { - int64_t loop_counter_ = 0; - - public: - ExampleRTThread(const char* name, cactus_rt::CyclicThreadConfig config) : CyclicThread(name, config) {} - - int64_t GetLoopCounter() const { - return loop_counter_; - } - - protected: - LoopControl Loop(int64_t /*now*/) noexcept final { - loop_counter_++; - return LoopControl::Continue; - } -}; - -int main() { - cactus_rt::CyclicThreadConfig config; - config.period_ns = 1'000'000; - config.cpu_affinity = std::vector{2}; - config.SetFifoScheduler(80); - - App app; - auto thread = app.CreateThread("ExampleRTThread", config); - - // Sets up the signal handlers for SIGINT and SIGTERM (by default). - cactus_rt::SetUpTerminationSignalHandler(); - - std::cout << "Testing RT loop for until CTRL+C\n"; - - app.Start(); - - // This function blocks until SIGINT or SIGTERM are received. - cactus_rt::WaitForAndHandleTerminationSignal(); - - // The following commented code is an example using the polling mechanism. - // while (!cactus_rt::HasTerminationSignalBeenReceived()) { - // std::this_thread::sleep_for(std::chrono::milliseconds(100)); - // } - - app.RequestStop(); - app.Join(); - - std::cout << "Number of loops executed: " << thread->GetLoopCounter() << "\n"; - return 0; -} diff --git a/examples/simple_example/README.md b/examples/simple_example/README.md index 3ad122f..e85e50f 100644 --- a/examples/simple_example/README.md +++ b/examples/simple_example/README.md @@ -1,4 +1,12 @@ `simple_example` ================ -The simplest real-time example to get started with. +This gives you the most basic scaffolding to get started with in a 1000-Hz +real-time application without any bells and whistles. + +To run this: + +```bash +$ make debug +$ sudo build/debug/examples/simple_example/rt_simple_example +``` diff --git a/examples/simple_example/main.cc b/examples/simple_example/main.cc index 70f3f73..bd89d20 100644 --- a/examples/simple_example/main.cc +++ b/examples/simple_example/main.cc @@ -29,14 +29,16 @@ class ExampleRTThread : public CyclicThread { * @brief This methods runs every loop, which for this particular example is every 1ms. * * @param elapsed_ns The number of nanoseconds elapsed since the App::Start was called. - * @return true if you want the thread to stop - * @return false if you want to thread to continue + * @return LoopControl::Continue if you want the thread to continue + * @return LoopControl::Stop if you want to thread to stop */ LoopControl Loop(int64_t elapsed_ns) noexcept final { // Code written in this function executes every 1 ms. // This demonstrates the usage of the quill logger. This emits a log message every 1s. LOG_INFO_LIMIT(std::chrono::seconds(1), Logger(), "Looping for {}", std::chrono::nanoseconds(elapsed_ns)); + + // Return LoopControl::Stop if you want the thread to stop. return LoopControl::Continue; } @@ -62,24 +64,30 @@ class ExampleRTThread : public CyclicThread { }; int main() { + // Sets up the signal handlers for SIGINT and SIGTERM (by default). + cactus_rt::SetUpTerminationSignalHandler(); + // We first create cactus_rt App object. App app; - // We then create a thread object. + // We then create a thread object. Threads should always be created via the + // App::CreateThread factory method. auto thread = app.CreateThread(); - constexpr unsigned int time = 5; - std::cout << "Testing RT loop for " << time << " seconds.\n"; - // Start the application, which starts all the registered threads (any thread // passed to App::RegisterThread) in the order they are registered. app.Start(); - // We let the application run for 5 seconds. - std::this_thread::sleep_for(std::chrono::seconds(time)); + std::cout << "App started\n"; + + // This function blocks until SIGINT or SIGTERM are received. + cactus_rt::WaitForAndHandleTerminationSignal(); + + std::cout << "Caught signal, requesting stop...\n"; - // We ask the application to stop, which stops all registered threads in the - // order they are registered. + // We ask the application to stop, which stops all threads in the order they + // are created. If you want the application to run indefinitely, remove this + // line. app.RequestStop(); // We wait until all threads registered are done here. diff --git a/examples/tracing_example/README.md b/examples/tracing_example/README.md index 11d8d45..b29693b 100644 --- a/examples/tracing_example/README.md +++ b/examples/tracing_example/README.md @@ -1,10 +1,37 @@ -Tracing example -=============== +`tracing_example` +================= -Designed to show off how to trace a real-time application. +Designed to show off how to trace a real-time application with the +real-time-safe tracing system. Features demonstrated: -1. Enable tracing. -2. Trace custom spans in the application. +1. Enable and disable tracing dynamically. +2. Trace custom code in your application with spans. 3. Setting the output file location. + +To run this: + +```bash +$ make debug +$ sudo build/debug/examples/tracing_example/rt_tracing_example +``` + +To visualize the trace, go to https://cactusdynamics.github.io/perfetto (or +https://ui.perfetto.dev which doesn't have as much bells and whistles) and load +the trace in `build/data1.perfetto` and `build/data2.perfetto`. + +You should be able to see something like this: + +![image](./perfetto-timeline1.png) + +By clicking on the `Loop` slice, filtering and sorting with the table that pops +up, then clicking on the ID of the longest `Loop` in the table will bring you to +the following view: + +![image](./perfetto-timeline2.png) + +Clicking on the _Latency_ tab on the side bar and selecting the right span will +get you something like the following: + +![image](./perfetto-hist.png) diff --git a/examples/tracing_example/main.cc b/examples/tracing_example/main.cc index 8e3701e..605f4b3 100644 --- a/examples/tracing_example/main.cc +++ b/examples/tracing_example/main.cc @@ -6,47 +6,33 @@ using cactus_rt::App; using cactus_rt::CyclicThread; -void WasteTime(std::chrono::microseconds duration) { +// A custom function to waste some time so we generate good looking traces. +void WasteTime(std::chrono::nanoseconds duration) { const auto start = cactus_rt::NowNs(); - auto duration_ns = duration.count() * 1000; + const auto duration_ns = duration.count(); while (cactus_rt::NowNs() - start < duration_ns) { } } class ExampleRTThread : public CyclicThread { - int64_t loop_counter_ = 0; - - static cactus_rt::CyclicThreadConfig CreateThreadConfig() { - cactus_rt::CyclicThreadConfig thread_config; - thread_config.period_ns = 1'000'000; - thread_config.cpu_affinity = std::vector{2}; - thread_config.SetFifoScheduler(80); - - // thread_config.tracer_config.trace_sleep = true; - thread_config.tracer_config.trace_wakeup_latency = true; - return thread_config; - } + int64_t last_overrun_ = 0; // A variable so we can generate an artificial loop overrun. public: - ExampleRTThread() : CyclicThread("ExampleRTThread", CreateThreadConfig()) {} - - int64_t GetLoopCounter() const { - return loop_counter_; - } + ExampleRTThread() : CyclicThread("ExampleRTThread", CreateConfig()) {} protected: - LoopControl Loop(int64_t /*now*/) noexcept final { - loop_counter_++; - if (loop_counter_ % 1000 == 0) { - LOG_INFO(Logger(), "Loop {}", loop_counter_); - } - + LoopControl Loop(int64_t elapsed_ns) noexcept final { + // If this is a robot controller, we can imagine that the main loop function + // will call three functions: sense, plan, and act. Each of these function + // are traced with custom code in the function so we can see it in the trace. Sense(); Plan(); Act(); - // Cause an overrun every 1.5s to demonstrate the overrun detection and marking feature. - if (loop_counter_ % 1500 == 0) { + // Cause an overrun every 1.5s to demonstrate the overrun detection and + // marking feature. We simulate some rogue process that takes 1200us. + if (elapsed_ns - last_overrun_ > 1'500'000'000) { + last_overrun_ = elapsed_ns; auto span = Tracer().WithSpan("RogueSegment", "app"); WasteTime(std::chrono::microseconds(1200)); } @@ -56,33 +42,72 @@ class ExampleRTThread : public CyclicThread { private: void Sense() noexcept { + // This uses RAII and emits a span with the duration that is the lifetime of + // the span variable. In this case, the span variable lasts for the duration + // of this function, which in this case we artificially constrain to 100us. + // + // The span has the name "Sense" and has the category of "app". All of these + // can be queried from the Perfetto web UI. auto span = Tracer().WithSpan("Sense", "app"); + + // Pretend that this code executes for 100us. WasteTime(std::chrono::microseconds(100)); } void Plan() noexcept { + // This uses RAII and emits a span with the duration that is the lifetime of + // the span variable. In this case, the span variable lasts for the duration + // of this function, which in this case we artificially constrain to 50us. + // + // The span has the name "Plan" and has the category of "app". All of these + // can be queried from the Perfetto web UI. auto span = Tracer().WithSpan("Plan", "app"); + + // Pretend that this code executes for 50us. WasteTime(std::chrono::microseconds(50)); } void Act() noexcept { + // This uses RAII and emits a span with the duration that is the lifetime of + // the span variable. In this case, the span variable lasts for the duration + // of this function, which in this case we artificially constrain to 75us. + // + // The span has the name "Act" and has the category of "app". All of these + // can be queried from the Perfetto web UI. auto span = Tracer().WithSpan("Act", "app"); + + // Pretend that this code executes for 75us. WasteTime(std::chrono::microseconds(75)); } -}; -class SecondRTThread : public CyclicThread { - static cactus_rt::CyclicThreadConfig CreateThreadConfig() { + static cactus_rt::CyclicThreadConfig CreateConfig() { + // This is a thread that runs every 1ms and is running with rtprio = 80. cactus_rt::CyclicThreadConfig thread_config; - thread_config.period_ns = 3'000'000; - thread_config.cpu_affinity = {1}; - thread_config.SetFifoScheduler(60); + thread_config.period_ns = 1'000'000; + thread_config.SetFifoScheduler(80); + + // cactus_rt will automatically trace the Loop latency as a span. It will + // also automatically mark any Loop that overruns the configured period with + // a marker. These can be turned off if not desired by uncommenting the + // following lines: + // thread_config.tracer_config.trace_loop = false; + // thread_config.tracer_config.trace_overrun = false; + // Uncomment this line if you want to see spans that indicates the sleeping + // period of // thread_config.tracer_config.trace_sleep = true; + + // This enables the tracing of wakeup latency, which shows the latency + // between when the thread is supposed to wake up and when the thread + // actually wakes up. thread_config.tracer_config.trace_wakeup_latency = true; return thread_config; } +}; +// Here we create a second real-time thread that runs at a lower frequency, to +// show that we can have multiple threads emitting trace data simultaneously. +class SecondRTThread : public CyclicThread { public: SecondRTThread() : CyclicThread("SecondRTThread", CreateThreadConfig()) {} @@ -91,11 +116,24 @@ class SecondRTThread : public CyclicThread { WasteTime(std::chrono::microseconds(2000)); return LoopControl::Continue; } + + static cactus_rt::CyclicThreadConfig CreateThreadConfig() { + // This is a thread that runs every 3ms and is running with rtprio = 60. + cactus_rt::CyclicThreadConfig thread_config; + thread_config.period_ns = 3'000'000; + thread_config.SetFifoScheduler(60); + + // thread_config.tracer_config.trace_sleep = true; + thread_config.tracer_config.trace_wakeup_latency = true; + return thread_config; + } }; int main() { cactus_rt::AppConfig app_config; - app_config.tracer_config.trace_aggregator_cpu_affinity = {0}; // doesn't work yet + // This should allow you to set where the background trace aggregator runs. + // TODO: this feature is not yet implemented in the framework. + app_config.tracer_config.trace_aggregator_cpu_affinity = {0}; App app("TracingExampleApp", app_config); @@ -105,24 +143,30 @@ int main() { std::cout << "Testing RT loop for 15 seconds with two trace sessions.\n"; // Start the first trace session before the app starts so we capture all - // events from the start. + // events from the start of the app. app.StartTraceSession("build/data1.perfetto"); + app.Start(); std::this_thread::sleep_for(std::chrono::seconds(5)); - // Stops the first session after 5 seconds. + + // Stops the first session after 5 seconds. The app remains running but no + // more trace data will be emitted. app.StopTraceSession(); + // Sleep another 5s without tracing data. std::this_thread::sleep_for(std::chrono::seconds(5)); - // Starts a second session after 5 seconds. + // Starts a second tracing session while the app is running and write the data + // to a different location. app.StartTraceSession("build/data2.perfetto"); std::this_thread::sleep_for(std::chrono::seconds(5)); // Stops the app. app.RequestStop(); app.Join(); - // Don't need to stop the trace session as the app destructor will take care of it. - std::cout << "Number of loops executed: " << thread1->GetLoopCounter() << "\n"; + // Note we don't need to explicitly stop the trace session as the app + // destructor will do it. If your app doesn't get destructed, you should + // probably call StopTraceSession manually. return 0; } diff --git a/examples/tracing_example/perfetto-hist.png b/examples/tracing_example/perfetto-hist.png new file mode 100644 index 0000000..e7cb373 Binary files /dev/null and b/examples/tracing_example/perfetto-hist.png differ diff --git a/examples/tracing_example/perfetto-timeline1.png b/examples/tracing_example/perfetto-timeline1.png new file mode 100644 index 0000000..173c10b Binary files /dev/null and b/examples/tracing_example/perfetto-timeline1.png differ diff --git a/examples/tracing_example/perfetto-timeline2.png b/examples/tracing_example/perfetto-timeline2.png new file mode 100644 index 0000000..7d1ff4c Binary files /dev/null and b/examples/tracing_example/perfetto-timeline2.png differ diff --git a/include/cactus_rt/signal_handler.h b/include/cactus_rt/signal_handler.h index 22b5ad5..2709a08 100644 --- a/include/cactus_rt/signal_handler.h +++ b/include/cactus_rt/signal_handler.h @@ -38,6 +38,8 @@ namespace cactus_rt { * cactus_rt::WaitForAndHandleTerminationSignal is unblocked. Further calls to * cactus_rt::HasTerminationSignalBeenReceived will return true. * + * Do not use this with the ROS2 app, as it already call it for you. + * * @param signals A vector of signals to catch. Default: SIGINT and SIGTERM. */ void SetUpTerminationSignalHandler(std::vector signals = {SIGINT, SIGTERM});