diff --git a/.github/workflows/create_docker_image.yml b/.github/workflows/create_docker_image.yml index 652e6ea..8e4e5f8 100644 --- a/.github/workflows/create_docker_image.yml +++ b/.github/workflows/create_docker_image.yml @@ -6,6 +6,8 @@ jobs: build_and_push_docker_image: name: build runs-on: ubuntu-latest + strategy: + fail-fast: false permissions: contents: read packages: write diff --git a/.github/workflows/industrial_ci.yml b/.github/workflows/industrial_ci.yml index d0f22c3..2971a45 100644 --- a/.github/workflows/industrial_ci.yml +++ b/.github/workflows/industrial_ci.yml @@ -6,6 +6,7 @@ jobs: industrial_ci: name: ROS ${{ matrix.ROS_DISTRO }} (${{ matrix.ROS_REPO }}) strategy: + fail-fast: false matrix: ROS_DISTRO: [rolling] ROS_REPO: [testing, main] diff --git a/.github/workflows/run_benchmarks.yml b/.github/workflows/run_benchmarks.yml index 9f87da6..be1f70f 100644 --- a/.github/workflows/run_benchmarks.yml +++ b/.github/workflows/run_benchmarks.yml @@ -1,36 +1,82 @@ -name: Run MoveIt Middleware Benchmarks and Push Results +name: Run MoveIt Middleware Benchmarks -on: [push] +on: [push, pull_request, workflow_dispatch] jobs: - run_middleware_benchmarks: - name: run_benchmarks + run_all_benchmarks: + name: run_all_benchmarks runs-on: ubuntu-latest + strategy: + fail-fast: false permissions: contents: write deployments: write container: image: ghcr.io/cihataltiparmak/moveit_middleware_benchmark:latest steps: - - name: run perception benchmark + - name: run benchmarks for rmw_fastrtps run: | - cd /ws + cd ${ROS_UNDERLAY}/.. . /opt/ros/rolling/setup.sh . install/setup.sh - ros2 launch moveit_middleware_benchmark scenario_perception_pipeline_benchmark.launch.py + sh src/moveit_middleware_benchmark/scripts/run_all_benchmarks.sh -i ./src/moveit_middleware_benchmark/middleware_configurations/rmw_fastrtps/config_rmw_fastrtps.sh -d /benchmark_results -m rmw_fastrtps_cpp + - name: run benchmarks for rmw_cyclonedds + run: | + cd ${ROS_UNDERLAY}/.. + . /opt/ros/rolling/setup.sh + . install/setup.sh + sh src/moveit_middleware_benchmark/scripts/run_all_benchmarks.sh -i ./src/moveit_middleware_benchmark/middleware_configurations/rmw_cyclonedds/config_rmw_cyclonedds.sh -d /benchmark_results -m rmw_cyclonedds_cpp - name: clone repo uses: actions/checkout@v3 - name: add to safe directory run: | git config --global --add safe.directory /__w/moveit_middleware_benchmark/moveit_middleware_benchmark - - name: push perception benchmark results to github pages + - name: push perception benchmark results for rmw_fastrtps to github pages + uses: benchmark-action/github-action-benchmark@v1 + with: + name: Perception Pipeline Benchmark + tool: 'googlecpp' + output-file-path: /benchmark_results/scenario_perception_pipeline/rmw_fastrtps_cpp.json + # Access token to deploy GitHub Pages branch + github-token: ${{ secrets.GITHUB_TOKEN }} + # Push and deploy GitHub pages branch automatically + auto-push: true + gh-pages-branch: "gh-pages" + benchmark-data-dir-path: "rmw_fastrtps" + - name: push simple service client benchmark results for rmw_fastrtps to github pages + uses: benchmark-action/github-action-benchmark@v1 + with: + name: Basic Service Client Benchmark + tool: 'googlecpp' + output-file-path: /benchmark_results/scenario_basic_service_client/rmw_fastrtps_cpp.json + # Access token to deploy GitHub Pages branch + github-token: ${{ secrets.GITHUB_TOKEN }} + # Push and deploy GitHub pages branch automatically + auto-push: true + gh-pages-branch: "gh-pages" + benchmark-data-dir-path: "rmw_fastrtps" + + - name: push perception benchmark results for rmw_cyclonedds to github pages + uses: benchmark-action/github-action-benchmark@v1 + with: + name: Perception Pipeline Benchmark + tool: 'googlecpp' + output-file-path: /benchmark_results/scenario_perception_pipeline/rmw_cyclonedds_cpp.json + # Access token to deploy GitHub Pages branch + github-token: ${{ secrets.GITHUB_TOKEN }} + # Push and deploy GitHub pages branch automatically + auto-push: true + gh-pages-branch: "gh-pages" + benchmark-data-dir-path: "rmw_cyclonedds" + - name: push simple service client benchmark results for rme_cyclonedds to github pages uses: benchmark-action/github-action-benchmark@v1 with: - name: Movet Middleware Benchmark Project Perception Pipeline Benchmark + name: Basic Service Client Benchmark tool: 'googlecpp' - output-file-path: /ws/middleware_benchmark_results.json + output-file-path: /benchmark_results/scenario_basic_service_client/rmw_cyclonedds_cpp.json # Access token to deploy GitHub Pages branch github-token: ${{ secrets.GITHUB_TOKEN }} # Push and deploy GitHub pages branch automatically auto-push: true gh-pages-branch: "gh-pages" + benchmark-data-dir-path: "rmw_cyclonedds" diff --git a/.github/workflows/test_moveit_middleware_benchmark_action.yml b/.github/workflows/test_moveit_middleware_benchmark_action.yml index b152d3f..3f717b5 100644 --- a/.github/workflows/test_moveit_middleware_benchmark_action.yml +++ b/.github/workflows/test_moveit_middleware_benchmark_action.yml @@ -6,6 +6,8 @@ jobs: moveit_middleware_benchmark_github_action_test: name: build runs-on: ubuntu-latest + strategy: + fail-fast: false steps: - uses: actions/checkout@v2 - uses: ./ diff --git a/CMakeLists.txt b/CMakeLists.txt index 06d4872..ac991b4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -14,6 +14,8 @@ find_package(dynmsg REQUIRED) find_package(nav_msgs REQUIRED) find_package(ament_index_cpp REQUIRED) find_package(yaml-cpp REQUIRED) +find_package(std_msgs REQUIRED) +find_package(example_interfaces REQUIRED) add_executable( scenario_perception_pipeline_benchmark_main @@ -28,7 +30,8 @@ ament_target_dependencies( "benchmark" "dynmsg" "nav_msgs" - "yaml-cpp") + "yaml-cpp" + "example_interfaces") target_include_directories( scenario_perception_pipeline_benchmark_main @@ -38,7 +41,30 @@ target_include_directories( target_link_libraries(scenario_perception_pipeline_benchmark_main PUBLIC "benchmark::benchmark" ${YAML_CPP_LIBRARIES}) +add_executable( + scenario_basic_service_client_benchmark_main + src/scenario_basic_service_client_benchmark_main.cpp + src/scenarios/scenario_basic_service_client.cpp) + +ament_target_dependencies( + scenario_basic_service_client_benchmark_main + PUBLIC + "moveit_ros_planning_interface" + "rclcpp" + "benchmark" + "std_msgs" + "example_interfaces") + +target_include_directories( + scenario_basic_service_client_benchmark_main + PUBLIC $ + $) + +target_link_libraries(scenario_basic_service_client_benchmark_main + PUBLIC "benchmark::benchmark" ${YAML_CPP_LIBRARIES}) + install(TARGETS scenario_perception_pipeline_benchmark_main + scenario_basic_service_client_benchmark_main DESTINATION lib/moveit_middleware_benchmark) install(DIRECTORY launch config DESTINATION share/moveit_middleware_benchmark) diff --git a/Dockerfile b/Dockerfile index ff7281a..a66f927 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM ros:rolling +FROM moveit/moveit2:rolling-source RUN apt-get update && \ apt install wget -y @@ -6,20 +6,17 @@ RUN apt-get update && \ RUN mkdir ws/src -p RUN . /opt/ros/rolling/setup.sh && \ - cd ws/src && \ - git clone https://github.com/CihatAltiparmak/moveit_middleware_benchmark.git -b development && \ + cd ${ROS_UNDERLAY}/../src && \ + git clone https://github.com/CihatAltiparmak/moveit_middleware_benchmark.git && \ vcs import < moveit_middleware_benchmark/moveit_middleware_benchmark.repos --recursive -RUN cd ws/src && \ - # git clone https://github.com/ros2/rmw_zenoh.git && \ - git clone https://github.com/ros2/rmw_cyclonedds.git - RUN . /opt/ros/rolling/setup.sh && \ - cd ws && \ + cd ${ROS_UNDERLAY}/.. && \ rosdep update --rosdistro=$ROS_DISTRO && \ apt-get update && \ + apt upgrade -y && \ rosdep install --from-paths src --ignore-src -r -y RUN . /opt/ros/rolling/setup.sh && \ - cd ws && \ + cd ${ROS_UNDERLAY}/.. && \ colcon build --mixin release --packages-skip test_dynmsg dynmsg_demo diff --git a/README.md b/README.md index 00b7a7e..69c3eed 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,7 @@ Under Construction This middleware benchmark tool aims to measure middleware effects on various scenarios like perception pipeline in MoveIt. There is a following list to see scenarios and how to measure the effects of middleware. * [Perception Pipeline](./docs/scenarios/perception_pipeline_benchmark.md) +* [Basic Service Client Works](./docs/scenarios/basic_service_client_benchmark.md) ## Getting Started diff --git a/docs/how_to_run.md b/docs/how_to_run.md index 2e894d6..3767bf8 100644 --- a/docs/how_to_run.md +++ b/docs/how_to_run.md @@ -1,3 +1,84 @@ +## Run All Benchmarks +Before running the following benchmarks, please read and apply the instructions outlined in the [How To Install](how_to_install.md) section for providing the necessary requirements. + +To run all benchmarks, just select your middleware implementation and go ahead with `run_all_benchmarks.sh` bash script. This command initially will run the script you chose. In this example, it's used the default config scripts which the necessary middleware configuration is applied. This property serves the functionality for the users to apply some custom configurations freely. + +```shell +# go to workspace this repository is built +cd ws +source /opt/ros/rolling/setup.sh +source install/setup.sh +# go to moveit_middleware_benchmark package's directory +cd src/moveit_middleware_benchmark +# conduct all benchmarks +sh src/moveit_middleware_benchmark/scripts/run_all_benchmarks.sh -i ./src/moveit_middleware_benchmark/middleware_configurations/rmw_cyclonedds/config.sh -d /benchmark_results +``` + +Let's explain all operations at `run_all_benchmarks.sh`. + +#### selection of initial script +``` +-i ./src/moveit_middleware_benchmark/middleware_configurations/rmw_cyclonedds/config.sh +``` + +This argument is for selecting the initial scripts to be run. These initial scripts are used for configuring middleware-specific settings for improved performance. For example, you can use the initial script to configure TCP settings for rmw_zenoh like below. + +```shell +echo "The configurations for rmw_zenoh_cpp is started!" +export RMW_IMPLEMENTATION=rmw_zenoh_cpp +sudo sysctl -w "net.ipv4.tcp_rmem=4096 4096 4096" +sudo sysctl -w "net.ipv4.tcp_wmem=4096 4096 4096" +sudo sysctl -w "net.ipv4.tcp_mem=4096 4096 4096" +echo "The configurations for rmw_zenoh_cpp is finished!" +``` + +#### directory selection to save benchmark results +``` +-d /benchmark_results +``` + +This argument indicates where the benchmark results are saved. For scenario_perception_benchmark and scenario_basic_service_client, the results of these scenarios are written in the shape of the below directory tree. It should be added that `run_all_benchmarks.sh` script uses json format to save the benchmark results. + +``` +benchmark_results/ +├── scenario_basic_service_client +│   ├── rmw_cyclonedds_cpp.json +│   ├── rmw_fastrtps_cpp.json +│   └── rmw_zenoh_cpp.json +└── scenario_perception_pipeline + ├── rmw_cyclonedds_cpp.json + ├── rmw_fastrtps_cpp.json + └── rmw_zenoh_cpp.json +``` + +## Plot Visualization of Benchmark Results + +After running `run_all_benchmarks.sh`, you can also visualize the box plots of benchmark results. Suppose that you have some benchmark results stored in below directory and the directory named `benchmark_results` is located in `ws` directory which this repository is built. + + +``` +benchmark_results/ +├── scenario_basic_service_client +│   ├── rmw_cyclonedds_cpp.json +│   ├── rmw_fastrtps_cpp.json +│   └── rmw_zenoh_cpp.json +└── scenario_perception_pipeline + ├── rmw_cyclonedds_cpp.json + ├── rmw_fastrtps_cpp.json + └── rmw_zenoh_cpp.json +``` + +Just give the directory of benchmark results as argument and then visualize the benchmark results in plot. +```shell +cd ws +python3 src/moveit_middleware_benchmark/scripts/box_plot_visualizer.py benchmark_results + +``` + +**NOTE THAT THE BELOW PICTURE DOESN'T PRESENT REAL RESULTS. IT'S JUST FOR SHOWCASE** + +![](./pictures/box_plot_example.png) + ## Scenarios ### [Perception Pipeline Benchmark](scenarios/perception_pipeline_benchmark.md) @@ -7,3 +88,9 @@ This benchmark measures the elapsed time by which the determined path is sent fo Firstly, `node` and `move_group_interface`in SetUp are created before each benchmark. `poses` inside `nav_msgs/msg/Path` is sent one by one to plan trajectory for robot. If planning is failed, only `failure_rate` is increased. If planning is successful, the trajectory_plan which move_group_server plan is sent via `move_group_interface` to start the execution of this planned trajectory. Then `success_number` is increased. For instance, the selected test_case includes 20 goal poses. These 20 goals is sent one by one to `move_group_server`. If the 5 goal poses out of 20 goal poses are failed, `success_number` equals 15 and `failure_number` equals 5. `success_number` and `failure_number` is important to observe the middlewares' behaviours. + +### [Basic Service Client Works Benchmark](scenarios/basic_service_client_benchmark.md) + +This benchmark measures the total elapsed time based on the time interval between sending the request by the client to the server and getting the response of server. This benchmark utilizes the [ros2/demos](https://github.com/ros2/demos) packages' [example server](https://github.com/ros2/demos/blob/rolling/demo_nodes_cpp/src/services/add_two_ints_server.cpp). + +In this benchmark scenario, the benchmarker node only has client interface. The necessary server for this client is run in [the launch file of this benchmark scenario](../launch/scenario_basic_service_client_benchmark.launch.py). Client sends a request to server and waits for the response from server. Client sends second request to server once the client receives response of first request from client. This actions are repeated `sending_request_number` times. You can configure this `sending_request_number` parameter in [this scenario's launch file]((../launch/scenario_basic_service_client_benchmark.launch.py)). diff --git a/docs/pictures/box_plot_example.png b/docs/pictures/box_plot_example.png new file mode 100644 index 0000000..eaa3808 Binary files /dev/null and b/docs/pictures/box_plot_example.png differ diff --git a/docs/scenarios/basic_service_client_benchmark.md b/docs/scenarios/basic_service_client_benchmark.md new file mode 100644 index 0000000..a0ba2a6 --- /dev/null +++ b/docs/scenarios/basic_service_client_benchmark.md @@ -0,0 +1,38 @@ +## How To Run Basic Service Client Benchmark + +Firstly, source your ros version. It's suggested to test with rolling version of ROS 2. + +For instance, to test with rmw_zenoh, start to zenoh router using following command in the terminal. +```sh +# go to your workspace +cd ws +# Be sure that ros2 daemon is killed. +pkill -9 -f ros && ros2 daemon stop +# Then start zenoh router +source /opt/ros/rolling/setup.bash +source install/setup.bash +export RMW_IMPLEMENTATION=rmw_zenoh_cpp +ros2 run rmw_zenoh_cpp rmw_zenohd +``` + +Select your rmw_implementation as `rmw_zenoh_cpp` and run the perception benchmark launch file in the another terminal. +```sh +# go to your workspace +cd ws +source /opt/ros/rolling/setup.bash +source install/setup.bash +export RMW_IMPLEMENTATION=rmw_zenoh_cpp # select your rmw_implementation to benchmark +ros2 launch moveit_middleware_benchmark scenario_basic_service_client_benchmark.launch.py +``` + +It will be defaultly benchmarked with 6 repetitions. It will be created the json file named `middleware_benchmark_results.json` for benchmarking results after finishing benchmark code execution. You can see the benchmark results in more detail inside this json file. + +If you want to customize your benchmark arguments or select different test case, you can use below command. + +```shell +ros2 launch moveit_middleware_benchmark scenario_basic_service_client_benchmark.launch.py benchmark_command_args:="--benchmark_out=middleware_benchmark_results.json --benchmark_out_format=json --benchmark_repetitions=1" sending_request_number:=50000 +``` + +## How to benchmark the basic service client execution + +The main idea here is to send some client request in `example_interfaces::srv::AddTwoInts` format to `add_two_ints_server` which is one of the examples of [ros2/demos package](https://github.com/ros2/demos) and then to measure the elapsed time by waiting response from server. This logic helps us to measure elapsed time between sending request and receiving response. diff --git a/include/moveit_middleware_benchmark/scenarios/scenario_basic_service_client.hpp b/include/moveit_middleware_benchmark/scenarios/scenario_basic_service_client.hpp new file mode 100644 index 0000000..31c98b7 --- /dev/null +++ b/include/moveit_middleware_benchmark/scenarios/scenario_basic_service_client.hpp @@ -0,0 +1,96 @@ +/********************************************************************* + * Software License Agreement (BSD License) + * + * Copyright (c) 2024, Cihat Kurtuluş Altıparmak + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * * Neither the name of PickNik Inc. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + *********************************************************************/ + +/* Author: Cihat Kurtuluş Altıparmak + Description: Benchmarking module to compare the effects of middlewares + against basic service client works + */ + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include + +namespace moveit +{ +namespace middleware_benchmark +{ + +class ScenarioBasicServiceClient +{ +public: + /** \brief Constructor + * \param [in] node The ros node for sending request and wait response from server + */ + ScenarioBasicServiceClient(const rclcpp::Node::SharedPtr& node); + + /** \brief the method to send the requests \e sending_request_number times + * \param [in] sending_request_number the number for how many request is sent by client + */ + void runTestCase(const int& sending_request_number); + +private: + rclcpp::Node::SharedPtr node_; + rclcpp::Client::SharedPtr client_; +}; + +class ScenarioBasicServiceClientFixture : public benchmark::Fixture +{ +public: + ScenarioBasicServiceClientFixture(); + + /** \brief This method runs once each benchmark starts + * \param [in] state + */ + void SetUp(::benchmark::State& /*state*/); + + /** \brief This method runs as soon as each benchmark finishes + * \param [in] state + */ + void TearDown(::benchmark::State& /*state*/); + +protected: + rclcpp::Node::SharedPtr node_; + /* The number to send request to server */ + int sending_request_number_; +}; + +} // namespace middleware_benchmark +} // namespace moveit diff --git a/launch/scenario_basic_service_client_benchmark.launch.py b/launch/scenario_basic_service_client_benchmark.launch.py new file mode 100644 index 0000000..9ab7b1d --- /dev/null +++ b/launch/scenario_basic_service_client_benchmark.launch.py @@ -0,0 +1,62 @@ +import os +from launch import LaunchDescription +from launch.substitutions import LaunchConfiguration +from ament_index_python.packages import get_package_share_directory +from launch.actions import ( + DeclareLaunchArgument, + ExecuteProcess, + Shutdown, + OpaqueFunction, +) +from launch_ros.substitutions import FindPackageShare +from launch_ros.actions import Node + + +def launch_setup(context, *args, **kwargs): + benchmark_command_args = context.perform_substitution( + LaunchConfiguration("benchmark_command_args") + ).split() + + sending_request_number = int( + context.perform_substitution(LaunchConfiguration("sending_request_number")) + ) + + add_two_ints_server_node = Node( + name="add_two_ints_server", + package="demo_nodes_cpp", + executable="add_two_ints_server", + ) + + benchmark_main_node = Node( + name="benchmark_main", + package="moveit_middleware_benchmark", + executable="scenario_basic_service_client_benchmark_main", + output="both", + arguments=benchmark_command_args, + parameters=[ + {"sending_request_number": sending_request_number}, + ], + on_exit=Shutdown(), + ) + + return [add_two_ints_server_node, benchmark_main_node] + + +def generate_launch_description(): + declared_arguments = [] + + benchmark_command_args = DeclareLaunchArgument( + "benchmark_command_args", + default_value="--benchmark_out=middleware_benchmark_results.json --benchmark_out_format=json --benchmark_repetitions=6", + description="Google Benchmark Tool Arguments", + ) + declared_arguments.append(benchmark_command_args) + + sending_request_number_arg = DeclareLaunchArgument( + "sending_request_number", default_value="10000" + ) + declared_arguments.append(sending_request_number_arg) + + return LaunchDescription( + declared_arguments + [OpaqueFunction(function=launch_setup)] + ) diff --git a/middleware_configurations/rmw_cyclonedds/config.sh b/middleware_configurations/rmw_cyclonedds/config.sh new file mode 100644 index 0000000..f9eba3b --- /dev/null +++ b/middleware_configurations/rmw_cyclonedds/config.sh @@ -0,0 +1,3 @@ +echo "The configurations for rmw_cyclonedds_cpp is started!" +export RMW_IMPLEMENTATION=rmw_cyclonedds_cpp +echo "The configurations for rmw_cyclonedds_cpp is finished!" diff --git a/middleware_configurations/rmw_fastrtps/config.sh b/middleware_configurations/rmw_fastrtps/config.sh new file mode 100644 index 0000000..5cfb878 --- /dev/null +++ b/middleware_configurations/rmw_fastrtps/config.sh @@ -0,0 +1,3 @@ +echo "The configurations for rmw_fastrtps_cpp is started!" +export RMW_IMPLEMENTATION=rmw_fastrtps_cpp +echo "The configurations for rmw_fastrtps_cpp is finished!" diff --git a/middleware_configurations/rmw_zenoh/config.sh b/middleware_configurations/rmw_zenoh/config.sh new file mode 100644 index 0000000..ad969e5 --- /dev/null +++ b/middleware_configurations/rmw_zenoh/config.sh @@ -0,0 +1,6 @@ +echo "The configurations for rmw_zenoh_cpp is started!" +export RMW_IMPLEMENTATION=rmw_zenoh_cpp +sudo sysctl -w "net.ipv4.tcp_rmem=4096 4096 4096" +sudo sysctl -w "net.ipv4.tcp_wmem=4096 4096 4096" +sudo sysctl -w "net.ipv4.tcp_mem=4096 4096 4096" +echo "The configurations for rmw_zenoh_cpp is finished!" diff --git a/moveit_middleware_benchmark.repos b/moveit_middleware_benchmark.repos index 53423c1..58ac283 100644 --- a/moveit_middleware_benchmark.repos +++ b/moveit_middleware_benchmark.repos @@ -1,8 +1,4 @@ repositories: - moveit_msgs: - type: git - url: https://github.com/moveit/moveit2.git - version: main moveit_resources: type: git url: https://github.com/moveit/moveit_resources.git @@ -11,6 +7,14 @@ repositories: type: git url: https://github.com/ros2/rmw_zenoh.git version: rolling + rmw_cyclonedds: + type: git + url: https://github.com/ros2/rmw_cyclonedds.git + version: rolling + rmw_fastrtps: + type: git + url: https://github.com/ros2/rmw_fastrtps.git + version: rolling dynamic_message_introspection: type: git url: https://github.com/osrf/dynamic_message_introspection.git diff --git a/package.xml b/package.xml index 13e3fe6..750482e 100644 --- a/package.xml +++ b/package.xml @@ -16,10 +16,12 @@ nav_msgs ament_index_cpp yaml-cpp + example_interfaces rmw_zenoh_cpp rmw_fastrtps_cpp rmw_cyclonedds_cpp + demo_nodes_cpp ament_lint_auto ament_lint_common diff --git a/scripts/box_plot_visualizer.py b/scripts/box_plot_visualizer.py new file mode 100644 index 0000000..1eaa980 --- /dev/null +++ b/scripts/box_plot_visualizer.py @@ -0,0 +1,77 @@ +import matplotlib.pyplot as plt +import numpy as np +import json +import os +import sys + +BENCHMARK_RESULTS_DIR = sys.argv[1] + +middleware_colors = { + "rmw_zenoh_cpp": "orange", + "rmw_cyclonedds_cpp": "peachpuff", + "rmw_fastrtps_cpp": "tomato", +} + +middleware_list = ["rmw_zenoh_cpp", "rmw_cyclonedds_cpp", "rmw_fastrtps_cpp"] + + +def read_benchmark_json(file_name): + benchmark_json_data = None + with open(file_name) as f: + benchmark_json_data = json.load(f) + return benchmark_json_data + + +def get_real_time_list_from_benchmark_json(benchmark_json_data): + real_time_list = [] + for benchmark_info in benchmark_json_data["benchmarks"]: + if benchmark_info["run_type"] == "iteration": + real_time_list.append(benchmark_info["real_time"]) + return real_time_list + + +def get_middleware_dataset_for_scenario(scenario_name): + middleware_datasets = [] + for middleware_name in middleware_list: + file_name = os.path.join( + BENCHMARK_RESULTS_DIR, scenario_name, f"{middleware_name}.json" + ) + benchmark_json_data = read_benchmark_json(file_name) + dataset = get_real_time_list_from_benchmark_json(benchmark_json_data) + middleware_datasets.append( + { + "name": middleware_name, + "dataset": dataset, + "color": middleware_colors[middleware_name], + } + ) + return middleware_datasets + + +def plot_dataset_of_scenario(plt, scenario_name, middleware_datasets): + labels = [] + colors = [] + datasets = [] + + for x in middleware_datasets: + labels.append(x["name"]) + colors.append(x["color"]) + datasets.append(x["dataset"]) + + fig, ax = plt.subplots() + ax.set_title(scenario_name) + ax.set_ylabel("real time (ns)") + ax.set_xlabel("middlewares") + + bplot = ax.boxplot(datasets, patch_artist=True, tick_labels=labels) + + # fill with colors + for patch, color in zip(bplot["boxes"], colors): + patch.set_facecolor(color) + + +for scenario_name in os.listdir(BENCHMARK_RESULTS_DIR): + middleware_datasets = get_middleware_dataset_for_scenario(scenario_name) + plot_dataset_of_scenario(plt, scenario_name, middleware_datasets) + +plt.show() diff --git a/scripts/run_all_benchmarks.sh b/scripts/run_all_benchmarks.sh new file mode 100644 index 0000000..800d8b6 --- /dev/null +++ b/scripts/run_all_benchmarks.sh @@ -0,0 +1,40 @@ +# Inspired from https://unix.stackexchange.com/questions/31414/how-can-i-pass-a-command-line-argument-into-a-shell-script +helpFunction() +{ + echo "" + echo "Usage: $0 -i initial_script -d benchmark_results_directory" + echo -i "\t-i initial_script to run once all benchmarks are started" + echo -d "\t-d the directory the benchmark results is saved" + exit 1 # Exit script after printing help +} + +while getopts "i:m:d:" opt +do + case "$opt" in + i ) initial_script="$OPTARG" ;; + d ) benchmark_results_directory="$OPTARG" ;; + ? ) helpFunction ;; # Print helpFunction in case parameter is non-existent + esac +done + +# Print helpFunction in case parameters are empty +if [ -z "$initial_script" ] || [ -z "$benchmark_results_directory" ] +then + echo "Some or all of the parameters are empty"; + helpFunction +fi + +echo "benchmark results directory is $benchmark_results_directory" + +echo "Benchmarking is starting!" +echo "Starting initial scripts before benchmarks run!" +. "$initial_script" +echo "Initial script has finished! Now starting to benchmark middleware with scenarios!" + +mkdir ${benchmark_results_directory}/scenario_basic_service_client -p +ros2 daemon stop +ros2 launch moveit_middleware_benchmark scenario_basic_service_client_benchmark.launch.py benchmark_command_args:="--benchmark_out=${benchmark_results_directory}/scenario_basic_service_client/${RMW_IMPLEMENTATION}.json --benchmark_out_format=json --benchmark_repetitions=6" + +mkdir ${benchmark_results_directory}/scenario_perception_pipeline -p +ros2 daemon stop +ros2 launch moveit_middleware_benchmark scenario_perception_pipeline_benchmark.launch.py benchmark_command_args:="--benchmark_out=${benchmark_results_directory}/scenario_perception_pipeline/${RMW_IMPLEMENTATION}.json --benchmark_out_format=json --benchmark_repetitions=6" diff --git a/src/scenario_basic_service_client_benchmark_main.cpp b/src/scenario_basic_service_client_benchmark_main.cpp new file mode 100644 index 0000000..6653d55 --- /dev/null +++ b/src/scenario_basic_service_client_benchmark_main.cpp @@ -0,0 +1,50 @@ +/********************************************************************* + * Software License Agreement (BSD License) + * + * Copyright (c) 2024, Cihat Kurtuluş Altıparmak + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * * Neither the name of PickNik Inc. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + *********************************************************************/ + +/* Author: Cihat Kurtuluş Altıparmak + Description: Benchmarking module to compare the effects of middlewares + against basic service client works + */ + +#include "moveit_middleware_benchmark/scenarios/scenario_basic_service_client.hpp" + +int main(int argc, char** argv) +{ + rclcpp::init(argc, argv); + benchmark::Initialize(&argc, argv); + benchmark::RunSpecifiedBenchmarks(); + benchmark::Shutdown(); + rclcpp::shutdown(); + return 0; +} diff --git a/src/scenarios/scenario_basic_service_client.cpp b/src/scenarios/scenario_basic_service_client.cpp new file mode 100644 index 0000000..5141b0c --- /dev/null +++ b/src/scenarios/scenario_basic_service_client.cpp @@ -0,0 +1,124 @@ +/********************************************************************* + * Software License Agreement (BSD License) + * + * Copyright (c) 2024, Cihat Kurtuluş Altıparmak + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * * Neither the name of PickNik Inc. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + *********************************************************************/ + +/* Author: Cihat Kurtuluş Altıparmak + Description: Benchmarking module to compare the effects of middlewares + against basic service client works + */ + +#include "moveit_middleware_benchmark/scenarios/scenario_basic_service_client.hpp" + +namespace moveit +{ +namespace middleware_benchmark +{ + +using namespace std::chrono_literals; + +ScenarioBasicServiceClient::ScenarioBasicServiceClient(const rclcpp::Node::SharedPtr& node) : node_(node) +{ + client_ = node_->create_client("add_two_ints"); + + if (!client_->wait_for_service(5s)) + { + RCLCPP_FATAL(node_->get_logger(), "Server is not available !"); + } + + // TODO @CihatAltiparmak : add this time stopings to + // perception pipeline benchmark as well + // It should be waited even if server is okay due + // to the fact that server sometimes doesn't response + // request we sent in case that client sent request once server is ready + std::this_thread::sleep_for(1s); +} + +void ScenarioBasicServiceClient::runTestCase(const int& sending_request_number) +{ + for (int i = 0; i < sending_request_number; i++) + { + auto request = std::make_shared(); + request->a = 5; + request->b = 4; + + auto result = client_->async_send_request(request); + + if (rclcpp::spin_until_future_complete(node_, result) == rclcpp::FutureReturnCode::SUCCESS) + { + RCLCPP_DEBUG(node_->get_logger(), "Response %d is successfully returned by server!", i); + } + else + { + RCLCPP_ERROR(node_->get_logger(), "Request %d failed!", i); + } + } +} + +ScenarioBasicServiceClientFixture::ScenarioBasicServiceClientFixture() +{ +} + +void ScenarioBasicServiceClientFixture::SetUp(::benchmark::State& /*state*/) +{ + if (node_.use_count() == 0) + { + node_ = std::make_shared("test_scenario_basic_service_client", + rclcpp::NodeOptions().automatically_declare_parameters_from_overrides(true)); + + node_->get_parameter("sending_request_number", sending_request_number_); + } +} + +void ScenarioBasicServiceClientFixture::TearDown(::benchmark::State& /*state*/) +{ + node_.reset(); +} + +BENCHMARK_DEFINE_F(ScenarioBasicServiceClientFixture, test_scenario_basic_service_client)(benchmark::State& st) +{ + for (auto _ : st) + { + // TODO @CihatAltiparmak : add this time stopings to + // perception pipeline benchmark as well + st.PauseTiming(); + auto sc = ScenarioBasicServiceClient(node_); + st.ResumeTiming(); + + sc.runTestCase(sending_request_number_); + } +} + +BENCHMARK_REGISTER_F(ScenarioBasicServiceClientFixture, test_scenario_basic_service_client); + +} // namespace middleware_benchmark +} // namespace moveit