From c7047edae56daf3448f7d7fbe76ad8604c573fcf Mon Sep 17 00:00:00 2001 From: Josh Bailey Date: Mon, 27 Nov 2023 19:46:03 +0000 Subject: [PATCH] Move image processing entirely to background thread, add max_rows parameter to image_inference. --- grc/iqtlabs_image_inference.block.yml | 7 +- include/gnuradio/iqtlabs/image_inference.h | 3 +- lib/image_inference_impl.cc | 99 +++++++++++-------- lib/image_inference_impl.h | 10 +- lib/retune_fft_impl.cc | 17 +--- lib/retune_fft_impl.h | 4 +- .../bindings/image_inference_python.cc | 3 +- python/iqtlabs/qa_image_inference.py | 1 + 8 files changed, 83 insertions(+), 61 deletions(-) diff --git a/grc/iqtlabs_image_inference.block.yml b/grc/iqtlabs_image_inference.block.yml index 54e3dea4..f9100095 100644 --- a/grc/iqtlabs_image_inference.block.yml +++ b/grc/iqtlabs_image_inference.block.yml @@ -9,7 +9,8 @@ templates: make: > iqtlabs.image_inference(${tag}, ${vlen}, ${x}, ${y}, ${image_dir}, ${convert_alpha}, ${norm_alpha}, ${norm_beta}, ${norm_type}, ${colormap}, - ${interpolation}, ${model_server}, ${model_name}, ${confidence}) + ${interpolation}, ${model_server}, ${model_name}, ${confidence}, + ${max_rows}) cpp_templates: includes: ['#include '] @@ -18,7 +19,7 @@ cpp_templates: this->${id} = gr::iqtlabs::image_inference::make(${tag}, ${vlen}, ${x}, ${y}, ${image_dir}, ${convert_alpha}, ${norm_alpha}, ${norm_beta}, ${norm_type}, ${colormap}, ${interpolation}, ${model_server}, - ${model_name}, ${confidence}); + ${model_name}, ${confidence}, ${max_rows}); link: ['libgnuradio-iqtlabs.so'] @@ -63,6 +64,8 @@ parameters: dtype: str - id: confidence dtype: float + - id: max_rows + dtype: int asserts: - ${ tag != "" } diff --git a/include/gnuradio/iqtlabs/image_inference.h b/include/gnuradio/iqtlabs/image_inference.h index 0c5bfaf9..7992303d 100644 --- a/include/gnuradio/iqtlabs/image_inference.h +++ b/include/gnuradio/iqtlabs/image_inference.h @@ -233,7 +233,8 @@ class IQTLABS_API image_inference : virtual public gr::block { double norm_alpha, double norm_beta, int norm_type, int colormap, int interpolation, int flip, double min_peak_points, const std::string &model_server, - const std::string &model_name, double confidence); + const std::string &model_name, double confidence, + int max_rows); }; } // namespace iqtlabs diff --git a/lib/image_inference_impl.cc b/lib/image_inference_impl.cc index 2c2363b1..d437699d 100644 --- a/lib/image_inference_impl.cc +++ b/lib/image_inference_impl.cc @@ -214,17 +214,16 @@ namespace gr { namespace iqtlabs { -image_inference::sptr -image_inference::make(const std::string &tag, int vlen, int x, int y, - const std::string &image_dir, double convert_alpha, - double norm_alpha, double norm_beta, int norm_type, - int colormap, int interpolation, int flip, - double min_peak_points, const std::string &model_server, - const std::string &model_name, double confidence) { +image_inference::sptr image_inference::make( + const std::string &tag, int vlen, int x, int y, + const std::string &image_dir, double convert_alpha, double norm_alpha, + double norm_beta, int norm_type, int colormap, int interpolation, int flip, + double min_peak_points, const std::string &model_server, + const std::string &model_name, double confidence, int max_rows) { return gnuradio::make_block_sptr( tag, vlen, x, y, image_dir, convert_alpha, norm_alpha, norm_beta, norm_type, colormap, interpolation, flip, min_peak_points, model_server, - model_name, confidence); + model_name, confidence, max_rows); } image_inference_impl::image_inference_impl( @@ -232,7 +231,7 @@ image_inference_impl::image_inference_impl( const std::string &image_dir, double convert_alpha, double norm_alpha, double norm_beta, int norm_type, int colormap, int interpolation, int flip, double min_peak_points, const std::string &model_server, - const std::string &model_name, double confidence) + const std::string &model_name, double confidence, int max_rows) : gr::block("image_inference", gr::io_signature::make(1 /* min inputs */, 1 /* max inputs */, vlen * sizeof(input_type)), @@ -243,8 +242,10 @@ image_inference_impl::image_inference_impl( norm_alpha_(norm_alpha), norm_beta_(norm_beta), norm_type_(norm_type), colormap_(colormap), interpolation_(interpolation), flip_(flip), min_peak_points_(min_peak_points), model_name_(model_name), - confidence_(confidence), running_(true), inference_connected_(false) { - points_buffer_.reset( + confidence_(confidence), max_rows_(max_rows), running_(true), + inference_connected_(false) { + points_buffer_ = new cv::Mat(cv::Size(vlen_, 0), CV_32F, cv::Scalar::all(0)); + normalized_buffer_.reset( new cv::Mat(cv::Size(vlen_, 0), CV_32F, cv::Scalar::all(0))); cmapped_buffer_.reset( new cv::Mat(cv::Size(vlen, 0), CV_8UC3, cv::Scalar::all(0))); @@ -266,7 +267,9 @@ image_inference_impl::image_inference_impl( } void image_inference_impl::delete_output_item_(output_item_type &output_item) { - delete output_item.image_buffer; + if (output_item.image_buffer) { + delete output_item.image_buffer; + } delete output_item.points_buffer; } @@ -282,50 +285,44 @@ image_inference_impl::~image_inference_impl() { while (!inference_q_.empty()) { delete_inference_(); } + delete points_buffer_; } void image_inference_impl::process_items_(size_t c, const input_type *&in) { - cv::Mat new_rows(cv::Size(vlen_, c), CV_32F, (void *)in); - points_buffer_->push_back(new_rows); + if (points_buffer_->rows < max_rows_) { + int clip_c = std::min(max_rows_ - points_buffer_->rows, (int)c); + cv::Mat new_rows(cv::Size(vlen_, clip_c), CV_32F, (void *)in); + double points_min, points_max; + cv::minMaxLoc(new_rows, &points_min, &points_max); + if (points_buffer_->empty()) { + points_min_ = points_min_; + points_max_ = points_max_; + } else { + points_min_ = std::min(points_min_, points_min); + points_max_ = std::max(points_max_, points_max); + } + points_buffer_->push_back(new_rows); + } in += (vlen_ * c); } void image_inference_impl::create_image_() { if (!points_buffer_->empty()) { - double points_min, points_max; - cv::minMaxLoc(*points_buffer_, &points_min, &points_max); - if (points_max > min_peak_points_) { + if (points_max_ > min_peak_points_) { output_item_type output_item; output_item.rx_freq = last_rx_freq_; output_item.ts = last_rx_time_; - output_item.points_min = points_min; - output_item.points_max = points_max; - output_item.image_buffer = - new cv::Mat(cv::Size(x_, y_), CV_8UC3, cv::Scalar::all(0)); - output_item.points_buffer = - new cv::Mat(cv::Size(vlen_, 0), CV_32F, cv::Scalar::all(0)); - if (flip_ == -1 || flip_ == 0 || flip_ == 1) { - cv::flip(*points_buffer_, *points_buffer_, flip_); - } - points_buffer_->copyTo(*output_item.points_buffer); - output_item.points_mean = cv::mean(*output_item.points_buffer)[0]; - this->d_logger->debug("rx_freq {} rx_time {} rows {}", last_rx_freq_, - last_rx_time_, points_buffer_->rows); - cv::normalize(*points_buffer_, *points_buffer_, norm_alpha_, norm_beta_, - norm_type_); - points_buffer_->convertTo(*cmapped_buffer_, CV_8UC3, convert_alpha_, 0); - cv::applyColorMap(*cmapped_buffer_, *cmapped_buffer_, colormap_); - cv::cvtColor(*cmapped_buffer_, *cmapped_buffer_, cv::COLOR_BGR2RGB); - cv::resize(*cmapped_buffer_, *resized_buffer_, cv::Size(x_, y_), - interpolation_); - cv::cvtColor(*resized_buffer_, *output_item.image_buffer, - cv::COLOR_RGB2BGR); + output_item.points_min = points_min_; + output_item.points_max = points_max_; + output_item.points_buffer = points_buffer_; + output_item.image_buffer = NULL; if (!inference_q_.push(output_item)) { d_logger->error("inference request queue full, size {}", MAX_INFERENCE); delete_output_item_(output_item); } } - points_buffer_->resize(0); + points_buffer_ = + new cv::Mat(cv::Size(vlen_, 0), CV_32F, cv::Scalar::all(0)); } } @@ -367,6 +364,24 @@ void image_inference_impl::get_inference_() { inference_q_.pop(output_item); boost::scoped_ptr> encoded_buffer; + output_item.image_buffer = + new cv::Mat(cv::Size(x_, y_), CV_8UC3, cv::Scalar::all(0)); + output_item.points_mean = cv::mean(*output_item.points_buffer)[0]; + this->d_logger->debug("rx_freq {} rx_time {} rows {}", last_rx_freq_, + last_rx_time_, points_buffer_->rows); + cv::normalize(*output_item.points_buffer, *normalized_buffer_, norm_alpha_, + norm_beta_, norm_type_); + normalized_buffer_->convertTo(*cmapped_buffer_, CV_8UC3, convert_alpha_, 0); + cv::applyColorMap(*cmapped_buffer_, *cmapped_buffer_, colormap_); + cv::cvtColor(*cmapped_buffer_, *cmapped_buffer_, cv::COLOR_BGR2RGB); + cv::resize(*cmapped_buffer_, *resized_buffer_, cv::Size(x_, y_), + interpolation_); + cv::cvtColor(*resized_buffer_, *output_item.image_buffer, + cv::COLOR_RGB2BGR); + if (flip_ == -1 || flip_ == 0 || flip_ == 1) { + cv::flip(*output_item.image_buffer, *output_item.image_buffer, flip_); + } + nlohmann::json metadata_json; metadata_json["rssi_max"] = std::to_string(output_item.points_max); metadata_json["rssi_mean"] = std::to_string(output_item.points_mean); @@ -432,6 +447,10 @@ void image_inference_impl::get_inference_() { if (results.size()) { if (nlohmann::json::accept(results)) { + if (flip_ == -1 || flip_ == 0 || flip_ == 1) { + cv::flip(*output_item.points_buffer, *output_item.points_buffer, + flip_); + } nlohmann::json original_results_json = nlohmann::json::parse(results); nlohmann::json results_json = original_results_json; size_t rendered_predictions = 0; diff --git a/lib/image_inference_impl.h b/lib/image_inference_impl.h index b94ebc5e..e645d6bb 100644 --- a/lib/image_inference_impl.h +++ b/lib/image_inference_impl.h @@ -239,13 +239,16 @@ typedef struct output_item { class image_inference_impl : public image_inference, base_impl { private: - int x_, y_, vlen_, norm_type_, colormap_, interpolation_, flip_; + int x_, y_, vlen_, norm_type_, colormap_, interpolation_, flip_, max_rows_; uint64_t last_rx_freq_; double convert_alpha_, norm_alpha_, norm_beta_, last_rx_time_, min_peak_points_, confidence_; + double points_min_, points_max_; boost::lockfree::spsc_queue inference_q_{MAX_INFERENCE}; boost::lockfree::spsc_queue json_q_{MAX_INFERENCE}; - boost::scoped_ptr cmapped_buffer_, resized_buffer_, points_buffer_; + boost::scoped_ptr cmapped_buffer_, resized_buffer_, + normalized_buffer_; + cv::Mat *points_buffer_; std::string image_dir_; pmt::pmt_t tag_; std::deque out_buf_; @@ -271,7 +274,8 @@ class image_inference_impl : public image_inference, base_impl { double norm_alpha, double norm_beta, int norm_type, int colormap, int interpolation, int flip, double min_peak_points, const std::string &model_server, - const std::string &model_name, double confidence); + const std::string &model_name, double confidence, + int max_rows); ~image_inference_impl(); int general_work(int noutput_items, gr_vector_int &ninput_items, gr_vector_const_void_star &input_items, diff --git a/lib/retune_fft_impl.cc b/lib/retune_fft_impl.cc index 8fb0fc7b..4b25c862 100644 --- a/lib/retune_fft_impl.cc +++ b/lib/retune_fft_impl.cc @@ -322,9 +322,8 @@ void retune_fft_impl::sum_items_(const input_type *in) { } } -size_t retune_fft_impl::process_items_(size_t c, const input_type *&in, - const input_type *&fft_output) { - size_t fft_output_items = 0; +void retune_fft_impl::process_items_(size_t c, const input_type *&in, + const input_type *&fft_output) { for (size_t i = 0; i < c; ++i) { for (size_t j = 0; j < (vlen_ / nfft_); ++j, in += nfft_) { if (skip_fft_count_) { @@ -354,10 +353,10 @@ size_t retune_fft_impl::process_items_(size_t c, const input_type *&in, std::memcpy((void *)fft_output, (const void *)in, nfft_ * sizeof(input_type)); fft_output += nfft_; + produce(1, 1); write_items_(in); sum_items_(in); ++sample_count_; - ++fft_output_items; if (need_retune_(1)) { if (!pre_fft_) { retune_now_(); @@ -365,7 +364,6 @@ size_t retune_fft_impl::process_items_(size_t c, const input_type *&in, } } } - return fft_output_items; } void retune_fft_impl::forecast(int noutput_items, @@ -478,10 +476,9 @@ void retune_fft_impl::process_tags_(const input_type *in, size_t in_count, std::vector rx_times; get_tags_in_window(all_tags, 0, 0, in_count); get_tags(tag_, all_tags, rx_freq_tags, rx_times, in_count); - size_t fft_output_items = 0; if (rx_freq_tags.empty()) { - fft_output_items += process_items_(in_count, in, fft_output); + process_items_(in_count, in, fft_output); } else { for (size_t t = 0; t < rx_freq_tags.size(); ++t) { const auto &tag = rx_freq_tags[t]; @@ -490,7 +487,7 @@ void retune_fft_impl::process_tags_(const input_type *in, size_t in_count, in_first += rel; if (rel > 0) { - fft_output_items += process_items_(rel, in, fft_output); + process_items_(rel, in, fft_output); } const uint64_t rx_freq = (uint64_t)pmt::to_double(tag.value); @@ -502,8 +499,6 @@ void retune_fft_impl::process_tags_(const input_type *in, size_t in_count, process_buckets_(rx_freq, rx_time); } } - - produce(1, fft_output_items); } int retune_fft_impl::general_work(int noutput_items, @@ -518,7 +513,6 @@ int retune_fft_impl::general_work(int noutput_items, std::copy(from, to, out); out_buf_.erase(from, to); produce(0, leftover); - produce(1, 0); return WORK_CALLED_PRODUCE; } @@ -531,7 +525,6 @@ int retune_fft_impl::general_work(int noutput_items, size_t in_first = nitems_read(0); process_tags_(in, in_count, in_first, fft_output); consume_each(in_count); - produce(0, 0); return WORK_CALLED_PRODUCE; } diff --git a/lib/retune_fft_impl.h b/lib/retune_fft_impl.h index c628e871..e7635ba7 100644 --- a/lib/retune_fft_impl.h +++ b/lib/retune_fft_impl.h @@ -224,8 +224,8 @@ class retune_fft_impl : public retune_fft, base_impl, retuner_impl { void retune_now_(); void write_items_(const input_type *in); void sum_items_(const input_type *in); - size_t process_items_(size_t c, const input_type *&in, - const input_type *&fft_output); + void process_items_(size_t c, const input_type *&in, + const input_type *&fft_output); void output_buckets_(const std::string &name, const std::list> &buckets, std::stringstream &ss); diff --git a/python/iqtlabs/bindings/image_inference_python.cc b/python/iqtlabs/bindings/image_inference_python.cc index f315ff87..12c16802 100644 --- a/python/iqtlabs/bindings/image_inference_python.cc +++ b/python/iqtlabs/bindings/image_inference_python.cc @@ -14,7 +14,7 @@ /* BINDTOOL_GEN_AUTOMATIC(0) */ /* BINDTOOL_USE_PYGCCXML(0) */ /* BINDTOOL_HEADER_FILE(image_inference.h) */ -/* BINDTOOL_HEADER_FILE_HASH(95fb94fd34e1930b2cdc80357e966443) */ +/* BINDTOOL_HEADER_FILE_HASH(b39ddd6fde28ff8492f4a3e4bcec592a) */ /***********************************************************************************/ #include @@ -55,6 +55,7 @@ void bind_image_inference(py::module& m) py::arg("model_server"), py::arg("model_name"), py::arg("confidence"), + py::arg("max_rows"), D(image_inference, make)) diff --git a/python/iqtlabs/qa_image_inference.py b/python/iqtlabs/qa_image_inference.py index e4bf3dcf..01d6948d 100755 --- a/python/iqtlabs/qa_image_inference.py +++ b/python/iqtlabs/qa_image_inference.py @@ -286,6 +286,7 @@ def test_instance(self): f"localhost:{port}", model_name, 0.8, + 1024, ) c2r = blocks.complex_to_real(1) stream2vector = blocks.stream_to_vector(gr.sizeof_float, fft_size)