From 7e031d6e1c62defd5175996aa7646cfb3f3fe718 Mon Sep 17 00:00:00 2001 From: FaaizHaikal Date: Wed, 28 Feb 2024 20:06:50 +0700 Subject: [PATCH 01/10] enhancement: add image mat subscriber --- data/color_classifier.json | 14 ++++++++++++++ include/ninshiki_cpp/node/ninshiki_cpp_node.hpp | 5 +++-- src/ninshiki_cpp/detector/color_detector.cpp | 4 ++-- src/ninshiki_cpp/node/ninshiki_cpp_node.cpp | 10 +++++++--- src/ninshiki_cpp_detector.cpp | 2 +- 5 files changed, 27 insertions(+), 8 deletions(-) create mode 100644 data/color_classifier.json diff --git a/data/color_classifier.json b/data/color_classifier.json new file mode 100644 index 0000000..8b2c44f --- /dev/null +++ b/data/color_classifier.json @@ -0,0 +1,14 @@ +{ + "ball": { + "min_hsv": [112, 30, 20], + "max_hsv": [172, 100, 100] + }, + "field": { + "min_hsv": [61, 10, 0], + "max_hsv": [109, 100, 100] + }, + "line": { + "min_hsv": [180, 0, 55], + "max_hsv": [260, 60, 100] + } +} diff --git a/include/ninshiki_cpp/node/ninshiki_cpp_node.hpp b/include/ninshiki_cpp/node/ninshiki_cpp_node.hpp index 790b559..7f02dc0 100644 --- a/include/ninshiki_cpp/node/ninshiki_cpp_node.hpp +++ b/include/ninshiki_cpp/node/ninshiki_cpp_node.hpp @@ -59,18 +59,19 @@ class NinshikiCppNode private: using Contours = ninshiki_interfaces::msg::Contours; using DetectedObjects = ninshiki_interfaces::msg::DetectedObjects; - using Image = shisen_interfaces::msg::Image; + using Image = sensor_msgs::msg::Image; rclcpp::Node::SharedPtr node; rclcpp::TimerBase::SharedPtr node_timer; rclcpp::Publisher::SharedPtr detected_object_publisher; rclcpp::Publisher::SharedPtr field_segmentation_publisher; + rclcpp::Subscription::SharedPtr image_subscriber; std::shared_ptr dnn_detection; std::shared_ptr color_detection; std::shared_ptr lbp_detection; - std::shared_ptr image_provider; + std::shared_ptr camera_node; cv::Mat received_frame; cv::Mat hsv_frame; diff --git a/src/ninshiki_cpp/detector/color_detector.cpp b/src/ninshiki_cpp/detector/color_detector.cpp index 1b44977..f0742fe 100644 --- a/src/ninshiki_cpp/detector/color_detector.cpp +++ b/src/ninshiki_cpp/detector/color_detector.cpp @@ -133,8 +133,8 @@ bool ColorDetector::sync_configuration() cv::Mat ColorDetector::classify(cv::Mat input) { - int h_min = (min_hue * 180) / 360; - int h_max = (max_hue * 180) / 360; + int h_min = (min_hue * 255) / 360; + int h_max = (max_hue * 255) / 360; int s_min = (min_saturation * 255) / 100; int s_max = (max_saturation * 255) / 100; diff --git a/src/ninshiki_cpp/node/ninshiki_cpp_node.cpp b/src/ninshiki_cpp/node/ninshiki_cpp_node.cpp index b799f62..75ba01c 100644 --- a/src/ninshiki_cpp/node/ninshiki_cpp_node.cpp +++ b/src/ninshiki_cpp/node/ninshiki_cpp_node.cpp @@ -38,14 +38,18 @@ NinshikiCppNode::NinshikiCppNode( field_segmentation_publisher = node->create_publisher( get_node_prefix() + "/color_detection", 10); - image_provider = std::make_shared(options); + image_subscriber = + node->create_subscription("camera/image", 10, [this](const Image::SharedPtr message) { + if (!message->data.empty()) { + received_frame = cv_bridge::toCvShare(message, "bgr8")->image; + } + }); node_timer = node->create_wall_timer( std::chrono::milliseconds(frequency), [this]() { - image_provider->update_mat(); - received_frame = image_provider->get_mat(); if (!received_frame.empty()) { + cv::cvtColor(received_frame, hsv_frame, cv::COLOR_BGR2HSV); publish(); } } diff --git a/src/ninshiki_cpp_detector.cpp b/src/ninshiki_cpp_detector.cpp index 5d7624d..dcff328 100644 --- a/src/ninshiki_cpp_detector.cpp +++ b/src/ninshiki_cpp_detector.cpp @@ -32,7 +32,7 @@ int main(int argc, char ** argv) shisen_cpp::Options options; // Default Value - std::string path = ""; + std::string path = "./src/ninshiki_cpp/data"; std::string topic_name = ""; std::string detection_method = "yolo"; int gpu = 0; From d40e25e49a24566fe4f5266574e7ec611f6151aa Mon Sep 17 00:00:00 2001 From: FaaizHaikal Date: Wed, 28 Feb 2024 20:17:38 +0700 Subject: [PATCH 02/10] fix: remove unused variable --- include/ninshiki_cpp/node/ninshiki_cpp_node.hpp | 1 - 1 file changed, 1 deletion(-) diff --git a/include/ninshiki_cpp/node/ninshiki_cpp_node.hpp b/include/ninshiki_cpp/node/ninshiki_cpp_node.hpp index 7f02dc0..70345bf 100644 --- a/include/ninshiki_cpp/node/ninshiki_cpp_node.hpp +++ b/include/ninshiki_cpp/node/ninshiki_cpp_node.hpp @@ -71,7 +71,6 @@ class NinshikiCppNode std::shared_ptr dnn_detection; std::shared_ptr color_detection; std::shared_ptr lbp_detection; - std::shared_ptr camera_node; cv::Mat received_frame; cv::Mat hsv_frame; From 49c02aab6a5de33fca9fe11425e07d26e52c8d24 Mon Sep 17 00:00:00 2001 From: FaaizHaikal Date: Fri, 1 Mar 2024 21:05:08 +0700 Subject: [PATCH 03/10] fix: remove unused variable and dependencies --- CMakeLists.txt | 6 +++--- data/color_classifier.json | 14 -------------- include/ninshiki_cpp/node/ninshiki_cpp_node.hpp | 4 ++-- package.xml | 2 +- src/ninshiki_cpp/node/ninshiki_cpp_node.cpp | 6 +++--- src/ninshiki_cpp_detector.cpp | 9 ++------- 6 files changed, 11 insertions(+), 30 deletions(-) delete mode 100644 data/color_classifier.json diff --git a/CMakeLists.txt b/CMakeLists.txt index cea61be..d96d876 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -19,7 +19,7 @@ find_package(ninshiki_interfaces REQUIRED) find_package(OpenCV REQUIRED) find_package(rclcpp REQUIRED) find_package(shisen_cpp REQUIRED) -find_package(shisen_interfaces REQUIRED) +find_package(sensor_msgs REQUIRED) add_library(${PROJECT_NAME} SHARED "src/${PROJECT_NAME}/detector/color_detector.cpp" @@ -41,7 +41,7 @@ ament_target_dependencies(${PROJECT_NAME} OpenCV rclcpp shisen_cpp - shisen_interfaces) + sensor_msgs) install(DIRECTORY "include" DESTINATION ".") @@ -72,7 +72,7 @@ ament_export_dependencies( OpenCV rclcpp shisen_cpp - shisen_interfaces) + sensor_msgs) ament_export_include_directories("include") ament_export_libraries(${PROJECT_NAME}) ament_package() diff --git a/data/color_classifier.json b/data/color_classifier.json deleted file mode 100644 index 8b2c44f..0000000 --- a/data/color_classifier.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "ball": { - "min_hsv": [112, 30, 20], - "max_hsv": [172, 100, 100] - }, - "field": { - "min_hsv": [61, 10, 0], - "max_hsv": [109, 100, 100] - }, - "line": { - "min_hsv": [180, 0, 55], - "max_hsv": [260, 60, 100] - } -} diff --git a/include/ninshiki_cpp/node/ninshiki_cpp_node.hpp b/include/ninshiki_cpp/node/ninshiki_cpp_node.hpp index 70345bf..f6deb2b 100644 --- a/include/ninshiki_cpp/node/ninshiki_cpp_node.hpp +++ b/include/ninshiki_cpp/node/ninshiki_cpp_node.hpp @@ -29,12 +29,12 @@ #include #include "rclcpp/rclcpp.hpp" +#include "sensor_msgs/msg/image.hpp" #include "ninshiki_cpp/detector/color_detector.hpp" #include "ninshiki_cpp/detector/detector.hpp" #include "ninshiki_cpp/detector/dnn_detector.hpp" #include "ninshiki_cpp/detector/lbp_detector.hpp" #include "ninshiki_interfaces/msg/detected_objects.hpp" -#include "shisen_interfaces/msg/image.hpp" #include "shisen_cpp/shisen_cpp.hpp" namespace ninshiki_cpp::node @@ -48,7 +48,7 @@ class NinshikiCppNode using LBPDetector = ninshiki_cpp::detector::LBPDetector; NinshikiCppNode( - rclcpp::Node::SharedPtr node, std::string topic_name, + rclcpp::Node::SharedPtr node, int frequency, shisen_cpp::Options options); void publish(); void set_detection( diff --git a/package.xml b/package.xml index 1043754..8829a77 100644 --- a/package.xml +++ b/package.xml @@ -13,7 +13,7 @@ nlohmann-json-dev rclcpp shisen_cpp - shisen_interfaces + sensor_msgs ament_lint_auto ament_lint_common diff --git a/src/ninshiki_cpp/node/ninshiki_cpp_node.cpp b/src/ninshiki_cpp/node/ninshiki_cpp_node.cpp index 75ba01c..db757c6 100644 --- a/src/ninshiki_cpp/node/ninshiki_cpp_node.cpp +++ b/src/ninshiki_cpp/node/ninshiki_cpp_node.cpp @@ -29,9 +29,9 @@ namespace ninshiki_cpp::node { NinshikiCppNode::NinshikiCppNode( - rclcpp::Node::SharedPtr node, std::string topic_name, + rclcpp::Node::SharedPtr node, int frequency, shisen_cpp::Options options) -: node(node), dnn_detection(nullptr), color_detection(nullptr) +: node(node), dnn_detection(nullptr), color_detection(nullptr), lbp_detection(nullptr) { detected_object_publisher = node->create_publisher( get_node_prefix() + "/dnn_detection", 10); @@ -68,7 +68,7 @@ void NinshikiCppNode::publish() detected_object_publisher->publish(lbp_detection->detection_result); // Clear detection_result - // received_frame.release(); + received_frame.release(); dnn_detection->detection_result.detected_objects.clear(); color_detection->detection_result.contours.clear(); lbp_detection->detection_result.detected_objects.clear(); diff --git a/src/ninshiki_cpp_detector.cpp b/src/ninshiki_cpp_detector.cpp index dcff328..001ac57 100644 --- a/src/ninshiki_cpp_detector.cpp +++ b/src/ninshiki_cpp_detector.cpp @@ -20,8 +20,6 @@ #include #include -#include -#include #include #include @@ -32,7 +30,7 @@ int main(int argc, char ** argv) shisen_cpp::Options options; // Default Value - std::string path = "./src/ninshiki_cpp/data"; + std::string path = ""; std::string topic_name = ""; std::string detection_method = "yolo"; int gpu = 0; @@ -95,9 +93,6 @@ int main(int argc, char ** argv) return 1; } } else if (pos == 0) { - topic_name = arg; - ++pos; - } else if (pos == 1) { path = arg; ++pos; } @@ -109,7 +104,7 @@ int main(int argc, char ** argv) auto node = std::make_shared("ninshiki_cpp"); auto ninshiki_cpp_node = std::make_shared( - node, topic_name, frequency, options); + node, frequency, options); auto dnn_detection = std::make_shared(gpu, myriad); auto color_detection = std::make_shared(); From 2f6871d4187a46b8f3388cc2164571778626b0ec Mon Sep 17 00:00:00 2001 From: FaaizHaikal Date: Fri, 1 Mar 2024 21:08:59 +0700 Subject: [PATCH 04/10] fix: add cv_bridge --- CMakeLists.txt | 7 +++++-- package.xml | 1 + src/ninshiki_cpp/node/ninshiki_cpp_node.cpp | 1 + 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index d96d876..ccbc92a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -20,6 +20,7 @@ find_package(OpenCV REQUIRED) find_package(rclcpp REQUIRED) find_package(shisen_cpp REQUIRED) find_package(sensor_msgs REQUIRED) +find_package(cv_bridge REQUIRED) add_library(${PROJECT_NAME} SHARED "src/${PROJECT_NAME}/detector/color_detector.cpp" @@ -41,7 +42,8 @@ ament_target_dependencies(${PROJECT_NAME} OpenCV rclcpp shisen_cpp - sensor_msgs) + sensor_msgs + cv_bridge) install(DIRECTORY "include" DESTINATION ".") @@ -72,7 +74,8 @@ ament_export_dependencies( OpenCV rclcpp shisen_cpp - sensor_msgs) + sensor_msgs + cv_bridge) ament_export_include_directories("include") ament_export_libraries(${PROJECT_NAME}) ament_package() diff --git a/package.xml b/package.xml index 8829a77..91d13d3 100644 --- a/package.xml +++ b/package.xml @@ -14,6 +14,7 @@ rclcpp shisen_cpp sensor_msgs + cv_bridge ament_lint_auto ament_lint_common diff --git a/src/ninshiki_cpp/node/ninshiki_cpp_node.cpp b/src/ninshiki_cpp/node/ninshiki_cpp_node.cpp index db757c6..a5cfff2 100644 --- a/src/ninshiki_cpp/node/ninshiki_cpp_node.cpp +++ b/src/ninshiki_cpp/node/ninshiki_cpp_node.cpp @@ -21,6 +21,7 @@ #include #include #include +#include #include "ninshiki_cpp/node/ninshiki_cpp_node.hpp" using namespace std::chrono_literals; From 5b6884bec88afb7bc6fce1056a05dd1d6da7e487 Mon Sep 17 00:00:00 2001 From: FaaizHaikal Date: Wed, 27 Mar 2024 21:00:55 +0700 Subject: [PATCH 05/10] fix: contour point message and save color config --- src/ninshiki_cpp/detector/color_detector.cpp | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/src/ninshiki_cpp/detector/color_detector.cpp b/src/ninshiki_cpp/detector/color_detector.cpp index c3cc3d0..673333e 100644 --- a/src/ninshiki_cpp/detector/color_detector.cpp +++ b/src/ninshiki_cpp/detector/color_detector.cpp @@ -99,9 +99,10 @@ bool ColorDetector::save_configuration() int max_hsv[] = {item.max_hue, item.max_saturation, item.max_value}; nlohmann::json color = { - {"name", item.name}, - {"min_hsv", min_hsv}, - {"max_hsv", max_hsv}, + {item.name, { + {"min_hsv", min_hsv}, + {"max_hsv", max_hsv} + }} }; config.push_back(color); @@ -189,10 +190,6 @@ void ColorDetector::find(cv::Mat binary_mat) void ColorDetector::detection(const cv::Mat & image) { - // Get width and height from image - double img_width = static_cast(image.cols); - double img_height = static_cast(image.rows); - // iterate every color in colors for (auto & color : colors) { color_name = color.name; @@ -213,8 +210,8 @@ void ColorDetector::detection(const cv::Mat & image) for (cv::Point & point : contour) { ninshiki_interfaces::msg::Point point_msg; - point_msg.x = static_cast(point.x) / img_width; - point_msg.y = static_cast(point.y) / img_height; + point_msg.x = static_cast(point.x); + point_msg.y = static_cast(point.y); contour_msg.name = color_name; contour_msg.contour.push_back(point_msg); From fae53e231cb56ace46f713bdcc8e93983ce8b16c Mon Sep 17 00:00:00 2001 From: FaaizHaikal Date: Wed, 27 Mar 2024 21:01:08 +0700 Subject: [PATCH 06/10] fix: fix argument handling --- src/ninshiki_cpp_detector.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/ninshiki_cpp_detector.cpp b/src/ninshiki_cpp_detector.cpp index 001ac57..48fd7ef 100644 --- a/src/ninshiki_cpp_detector.cpp +++ b/src/ninshiki_cpp_detector.cpp @@ -39,10 +39,11 @@ int main(int argc, char ** argv) const char * help_message = "Usage: ros2 run ninshiki_cpp detector\n" - "[topic] [--detector DETECTOR] [--GPU {0,1}] [--MYRIAD {0,1}]\n" + "[path] [--detector DETECTOR] [--GPU {0,1}] [--MYRIAD {0,1}]\n" "\n" + "" "Positional arguments:\n" - "topic specify topic name to subscribe\n" + "path path to detection configuration\n" "\n" "Optional arguments:\n" "-h, --help show this help message and exit\n" @@ -53,6 +54,9 @@ int main(int argc, char ** argv) // Handle arguments try { + if (argc < 2) { + throw std::runtime_error(""); + } int i = 1; int pos = 0; while (i < argc) { From b3ac0fcc9f3b8db3b6362726874ed03356c13509 Mon Sep 17 00:00:00 2001 From: FaaizHaikal Date: Wed, 27 Mar 2024 23:50:44 +0700 Subject: [PATCH 07/10] fix: fix detected object rect message --- src/ninshiki_cpp/detector/dnn_detector.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/ninshiki_cpp/detector/dnn_detector.cpp b/src/ninshiki_cpp/detector/dnn_detector.cpp index eff54d2..47d9d5a 100644 --- a/src/ninshiki_cpp/detector/dnn_detector.cpp +++ b/src/ninshiki_cpp/detector/dnn_detector.cpp @@ -164,10 +164,10 @@ void DnnDetector::detect_darknet(const cv::Mat & image, float conf_threshold, fl ninshiki_interfaces::msg::DetectedObject detection_object; detection_object.label = classes[class_ids[i]]; detection_object.score = confidences[i]; - detection_object.left = box.x / img_width; - detection_object.top = box.y / img_height; - detection_object.right = (box.x + box.width) / img_width; - detection_object.bottom = (box.y + box.height) / img_height; + detection_object.left = box.x; + detection_object.top = box.y; + detection_object.right = box.width; + detection_object.bottom = box.height; detection_result.detected_objects.push_back(detection_object); } From a5db624425c2e4549770c9d04b153a0c8f80d4f1 Mon Sep 17 00:00:00 2001 From: FaaizHaikal Date: Sun, 31 Mar 2024 12:49:35 +0700 Subject: [PATCH 08/10] fix: log empty arg error --- src/ninshiki_cpp_detector.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ninshiki_cpp_detector.cpp b/src/ninshiki_cpp_detector.cpp index 48fd7ef..23cc588 100644 --- a/src/ninshiki_cpp_detector.cpp +++ b/src/ninshiki_cpp_detector.cpp @@ -55,7 +55,8 @@ int main(int argc, char ** argv) // Handle arguments try { if (argc < 2) { - throw std::runtime_error(""); + std::cerr << "Argument needed!\n\n" << help_message << std::endl; + return 1; } int i = 1; int pos = 0; From 0ee0c167b281cecb68e8e5c03219601164399c4f Mon Sep 17 00:00:00 2001 From: FaaizHaikal Date: Mon, 1 Apr 2024 23:14:35 +0700 Subject: [PATCH 09/10] fix: use inference engine openvino for better computation --- src/ninshiki_cpp/detector/dnn_detector.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ninshiki_cpp/detector/dnn_detector.cpp b/src/ninshiki_cpp/detector/dnn_detector.cpp index 47d9d5a..164d286 100644 --- a/src/ninshiki_cpp/detector/dnn_detector.cpp +++ b/src/ninshiki_cpp/detector/dnn_detector.cpp @@ -55,7 +55,7 @@ DnnDetector::DnnDetector(bool gpu, bool myriad) net.setPreferableBackend(cv::dnn::DNN_BACKEND_OPENCV); net.setPreferableTarget(cv::dnn::DNN_TARGET_MYRIAD); } else { - net.setPreferableBackend(cv::dnn::DNN_BACKEND_OPENCV); + net.setPreferableBackend(cv::dnn::DNN_BACKEND_INFERENCE_ENGINE); net.setPreferableTarget(cv::dnn::DNN_TARGET_CPU); } } @@ -75,7 +75,7 @@ void DnnDetector::detect_darknet(const cv::Mat & image, float conf_threshold, fl // Create a 4D blob from a frame static cv::Mat blob; - cv::Size input_size = cv::Size(416, 416); + cv::Size input_size = cv::Size(320, 320); cv::dnn::blobFromImage(image, blob, 1.0, input_size, cv::Scalar(), true, false, CV_8U); net.setInput(blob, "", 0.00392, cv::Scalar(0, 0, 0, 0)); From 4093fb9d59ca5f23bb7bfe18b490b69e00cd67b4 Mon Sep 17 00:00:00 2001 From: FaaizHaikal Date: Fri, 5 Apr 2024 02:10:27 +0700 Subject: [PATCH 10/10] fix: turn of swapRB --- data/color_classifier.json | 14 -------------- src/ninshiki_cpp/detector/dnn_detector.cpp | 2 +- 2 files changed, 1 insertion(+), 15 deletions(-) delete mode 100644 data/color_classifier.json diff --git a/data/color_classifier.json b/data/color_classifier.json deleted file mode 100644 index 40fcfc1..0000000 --- a/data/color_classifier.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "ball": { - "min_hsv": [151, 40, 10], - "max_hsv": [179, 100, 100] - }, - "field": { - "min_hsv": [84, 10, 0], - "max_hsv": [116, 100, 100] - }, - "line": { - "min_hsv": [180, 0, 55], - "max_hsv": [260, 60, 100] - } -} diff --git a/src/ninshiki_cpp/detector/dnn_detector.cpp b/src/ninshiki_cpp/detector/dnn_detector.cpp index 164d286..475f7c5 100644 --- a/src/ninshiki_cpp/detector/dnn_detector.cpp +++ b/src/ninshiki_cpp/detector/dnn_detector.cpp @@ -76,7 +76,7 @@ void DnnDetector::detect_darknet(const cv::Mat & image, float conf_threshold, fl // Create a 4D blob from a frame static cv::Mat blob; cv::Size input_size = cv::Size(320, 320); - cv::dnn::blobFromImage(image, blob, 1.0, input_size, cv::Scalar(), true, false, CV_8U); + cv::dnn::blobFromImage(image, blob, 1.0, input_size, cv::Scalar(), false, false, CV_8U); net.setInput(blob, "", 0.00392, cv::Scalar(0, 0, 0, 0)); std::vector outs;