Merge pull request #148 from anarkiwi/opt

Move image processing entirely to background thread, add max_rows par…

grc/iqtlabs_image_inference.block.yml

@@ -9,7 +9,8 @@ templates:
   make: >
     iqtlabs.image_inference(${tag}, ${vlen}, ${x}, ${y}, ${image_dir},
     ${convert_alpha}, ${norm_alpha}, ${norm_beta}, ${norm_type}, ${colormap},
-    ${interpolation}, ${model_server}, ${model_name}, ${confidence})
+    ${interpolation}, ${model_server}, ${model_name}, ${confidence},
+    ${max_rows})
 
 cpp_templates:
   includes: ['#include <gnuradio/iqtlabs/image_inference.h>']
@@ -18,7 +19,7 @@ cpp_templates:
     this->${id} = gr::iqtlabs::image_inference::make(${tag}, ${vlen},
     ${x}, ${y}, ${image_dir}, ${convert_alpha}, ${norm_alpha}, ${norm_beta},
     ${norm_type}, ${colormap}, ${interpolation}, ${model_server},
-    ${model_name}, ${confidence});
+    ${model_name}, ${confidence}, ${max_rows});
   link: ['libgnuradio-iqtlabs.so']
 
 
@@ -63,6 +64,8 @@ parameters:
     dtype: str
   - id: confidence
     dtype: float
+  - id: max_rows
+    dtype: int
 
 asserts:
   - ${ tag != "" }

include/gnuradio/iqtlabs/image_inference.h

@@ -233,7 +233,8 @@ class IQTLABS_API image_inference : virtual public gr::block {
                    double norm_alpha, double norm_beta, int norm_type,
                    int colormap, int interpolation, int flip,
                    double min_peak_points, const std::string &model_server,
-                   const std::string &model_name, double confidence);
+                   const std::string &model_name, double confidence,
+                   int max_rows);
 };
 
 } // namespace iqtlabs

lib/image_inference_impl.cc

@@ -214,25 +214,24 @@
 namespace gr {
 namespace iqtlabs {
 
-image_inference::sptr
-image_inference::make(const std::string &tag, int vlen, int x, int y,
-                      const std::string &image_dir, double convert_alpha,
-                      double norm_alpha, double norm_beta, int norm_type,
-                      int colormap, int interpolation, int flip,
-                      double min_peak_points, const std::string &model_server,
-                      const std::string &model_name, double confidence) {
+image_inference::sptr image_inference::make(
+    const std::string &tag, int vlen, int x, int y,
+    const std::string &image_dir, double convert_alpha, double norm_alpha,
+    double norm_beta, int norm_type, int colormap, int interpolation, int flip,
+    double min_peak_points, const std::string &model_server,
+    const std::string &model_name, double confidence, int max_rows) {
   return gnuradio::make_block_sptr<image_inference_impl>(
       tag, vlen, x, y, image_dir, convert_alpha, norm_alpha, norm_beta,
       norm_type, colormap, interpolation, flip, min_peak_points, model_server,
-      model_name, confidence);
+      model_name, confidence, max_rows);
 }
 
 image_inference_impl::image_inference_impl(
     const std::string &tag, int vlen, int x, int y,
     const std::string &image_dir, double convert_alpha, double norm_alpha,
     double norm_beta, int norm_type, int colormap, int interpolation, int flip,
     double min_peak_points, const std::string &model_server,
-    const std::string &model_name, double confidence)
+    const std::string &model_name, double confidence, int max_rows)
     : gr::block("image_inference",
                 gr::io_signature::make(1 /* min inputs */, 1 /* max inputs */,
                                        vlen * sizeof(input_type)),
@@ -243,8 +242,10 @@ image_inference_impl::image_inference_impl(
       norm_alpha_(norm_alpha), norm_beta_(norm_beta), norm_type_(norm_type),
       colormap_(colormap), interpolation_(interpolation), flip_(flip),
       min_peak_points_(min_peak_points), model_name_(model_name),
-      confidence_(confidence), running_(true), inference_connected_(false) {
-  points_buffer_.reset(
+      confidence_(confidence), max_rows_(max_rows), running_(true),
+      inference_connected_(false) {
+  points_buffer_ = new cv::Mat(cv::Size(vlen_, 0), CV_32F, cv::Scalar::all(0));
+  normalized_buffer_.reset(
       new cv::Mat(cv::Size(vlen_, 0), CV_32F, cv::Scalar::all(0)));
   cmapped_buffer_.reset(
       new cv::Mat(cv::Size(vlen, 0), CV_8UC3, cv::Scalar::all(0)));
@@ -266,7 +267,9 @@ image_inference_impl::image_inference_impl(
 }
 
 void image_inference_impl::delete_output_item_(output_item_type &output_item) {
-  delete output_item.image_buffer;
+  if (output_item.image_buffer) {
+    delete output_item.image_buffer;
+  }
   delete output_item.points_buffer;
 }
 
@@ -282,50 +285,44 @@ image_inference_impl::~image_inference_impl() {
   while (!inference_q_.empty()) {
     delete_inference_();
   }
+  delete points_buffer_;
 }
 
 void image_inference_impl::process_items_(size_t c, const input_type *&in) {
-  cv::Mat new_rows(cv::Size(vlen_, c), CV_32F, (void *)in);
-  points_buffer_->push_back(new_rows);
+  if (points_buffer_->rows < max_rows_) {
+    int clip_c = std::min(max_rows_ - points_buffer_->rows, (int)c);
+    cv::Mat new_rows(cv::Size(vlen_, clip_c), CV_32F, (void *)in);
+    double points_min, points_max;
+    cv::minMaxLoc(new_rows, &points_min, &points_max);
+    if (points_buffer_->empty()) {
+      points_min_ = points_min_;
+      points_max_ = points_max_;
+    } else {
+      points_min_ = std::min(points_min_, points_min);
+      points_max_ = std::max(points_max_, points_max);
+    }
+    points_buffer_->push_back(new_rows);
+  }
   in += (vlen_ * c);
 }
 
 void image_inference_impl::create_image_() {
   if (!points_buffer_->empty()) {
-    double points_min, points_max;
-    cv::minMaxLoc(*points_buffer_, &points_min, &points_max);
-    if (points_max > min_peak_points_) {
+    if (points_max_ > min_peak_points_) {
       output_item_type output_item;
       output_item.rx_freq = last_rx_freq_;
       output_item.ts = last_rx_time_;
-      output_item.points_min = points_min;
-      output_item.points_max = points_max;
-      output_item.image_buffer =
-          new cv::Mat(cv::Size(x_, y_), CV_8UC3, cv::Scalar::all(0));
-      output_item.points_buffer =
-          new cv::Mat(cv::Size(vlen_, 0), CV_32F, cv::Scalar::all(0));
-      if (flip_ == -1 || flip_ == 0 || flip_ == 1) {
-        cv::flip(*points_buffer_, *points_buffer_, flip_);
-      }
-      points_buffer_->copyTo(*output_item.points_buffer);
-      output_item.points_mean = cv::mean(*output_item.points_buffer)[0];
-      this->d_logger->debug("rx_freq {} rx_time {} rows {}", last_rx_freq_,
-                            last_rx_time_, points_buffer_->rows);
-      cv::normalize(*points_buffer_, *points_buffer_, norm_alpha_, norm_beta_,
-                    norm_type_);
-      points_buffer_->convertTo(*cmapped_buffer_, CV_8UC3, convert_alpha_, 0);
-      cv::applyColorMap(*cmapped_buffer_, *cmapped_buffer_, colormap_);
-      cv::cvtColor(*cmapped_buffer_, *cmapped_buffer_, cv::COLOR_BGR2RGB);
-      cv::resize(*cmapped_buffer_, *resized_buffer_, cv::Size(x_, y_),
-                 interpolation_);
-      cv::cvtColor(*resized_buffer_, *output_item.image_buffer,
-                   cv::COLOR_RGB2BGR);
+      output_item.points_min = points_min_;
+      output_item.points_max = points_max_;
+      output_item.points_buffer = points_buffer_;
+      output_item.image_buffer = NULL;
       if (!inference_q_.push(output_item)) {
         d_logger->error("inference request queue full, size {}", MAX_INFERENCE);
         delete_output_item_(output_item);
       }
     }
-    points_buffer_->resize(0);
+    points_buffer_ =
+        new cv::Mat(cv::Size(vlen_, 0), CV_32F, cv::Scalar::all(0));
   }
 }
 
@@ -367,6 +364,24 @@ void image_inference_impl::get_inference_() {
     inference_q_.pop(output_item);
     boost::scoped_ptr<std::vector<unsigned char>> encoded_buffer;
 
+    output_item.image_buffer =
+        new cv::Mat(cv::Size(x_, y_), CV_8UC3, cv::Scalar::all(0));
+    output_item.points_mean = cv::mean(*output_item.points_buffer)[0];
+    this->d_logger->debug("rx_freq {} rx_time {} rows {}", last_rx_freq_,
+                          last_rx_time_, points_buffer_->rows);
+    cv::normalize(*output_item.points_buffer, *normalized_buffer_, norm_alpha_,
+                  norm_beta_, norm_type_);
+    normalized_buffer_->convertTo(*cmapped_buffer_, CV_8UC3, convert_alpha_, 0);
+    cv::applyColorMap(*cmapped_buffer_, *cmapped_buffer_, colormap_);
+    cv::cvtColor(*cmapped_buffer_, *cmapped_buffer_, cv::COLOR_BGR2RGB);
+    cv::resize(*cmapped_buffer_, *resized_buffer_, cv::Size(x_, y_),
+               interpolation_);
+    cv::cvtColor(*resized_buffer_, *output_item.image_buffer,
+                 cv::COLOR_RGB2BGR);
+    if (flip_ == -1 || flip_ == 0 || flip_ == 1) {
+      cv::flip(*output_item.image_buffer, *output_item.image_buffer, flip_);
+    }
+
     nlohmann::json metadata_json;
     metadata_json["rssi_max"] = std::to_string(output_item.points_max);
     metadata_json["rssi_mean"] = std::to_string(output_item.points_mean);
@@ -432,6 +447,10 @@ void image_inference_impl::get_inference_() {
 
       if (results.size()) {
         if (nlohmann::json::accept(results)) {
+          if (flip_ == -1 || flip_ == 0 || flip_ == 1) {
+            cv::flip(*output_item.points_buffer, *output_item.points_buffer,
+                     flip_);
+          }
           nlohmann::json original_results_json = nlohmann::json::parse(results);
           nlohmann::json results_json = original_results_json;
           size_t rendered_predictions = 0;

lib/image_inference_impl.h

@@ -239,13 +239,16 @@ typedef struct output_item {
 
 class image_inference_impl : public image_inference, base_impl {
 private:
-  int x_, y_, vlen_, norm_type_, colormap_, interpolation_, flip_;
+  int x_, y_, vlen_, norm_type_, colormap_, interpolation_, flip_, max_rows_;
   uint64_t last_rx_freq_;
   double convert_alpha_, norm_alpha_, norm_beta_, last_rx_time_,
       min_peak_points_, confidence_;
+  double points_min_, points_max_;
   boost::lockfree::spsc_queue<output_item_type> inference_q_{MAX_INFERENCE};
   boost::lockfree::spsc_queue<std::string> json_q_{MAX_INFERENCE};
-  boost::scoped_ptr<cv::Mat> cmapped_buffer_, resized_buffer_, points_buffer_;
+  boost::scoped_ptr<cv::Mat> cmapped_buffer_, resized_buffer_,
+      normalized_buffer_;
+  cv::Mat *points_buffer_;
   std::string image_dir_;
   pmt::pmt_t tag_;
   std::deque<output_type> out_buf_;
@@ -271,7 +274,8 @@ class image_inference_impl : public image_inference, base_impl {
                        double norm_alpha, double norm_beta, int norm_type,
                        int colormap, int interpolation, int flip,
                        double min_peak_points, const std::string &model_server,
-                       const std::string &model_name, double confidence);
+                       const std::string &model_name, double confidence,
+                       int max_rows);
   ~image_inference_impl();
   int general_work(int noutput_items, gr_vector_int &ninput_items,
                    gr_vector_const_void_star &input_items,

lib/retune_fft_impl.cc

@@ -322,9 +322,8 @@ void retune_fft_impl::sum_items_(const input_type *in) {
   }
 }
 
-size_t retune_fft_impl::process_items_(size_t c, const input_type *&in,
-                                       const input_type *&fft_output) {
-  size_t fft_output_items = 0;
+void retune_fft_impl::process_items_(size_t c, const input_type *&in,
+                                     const input_type *&fft_output) {
   for (size_t i = 0; i < c; ++i) {
     for (size_t j = 0; j < (vlen_ / nfft_); ++j, in += nfft_) {
       if (skip_fft_count_) {
@@ -354,18 +353,17 @@ size_t retune_fft_impl::process_items_(size_t c, const input_type *&in,
       std::memcpy((void *)fft_output, (const void *)in,
                   nfft_ * sizeof(input_type));
       fft_output += nfft_;
+      produce(1, 1);
       write_items_(in);
       sum_items_(in);
       ++sample_count_;
-      ++fft_output_items;
       if (need_retune_(1)) {
         if (!pre_fft_) {
           retune_now_();
         }
       }
     }
   }
-  return fft_output_items;
 }
 
 void retune_fft_impl::forecast(int noutput_items,
@@ -478,10 +476,9 @@ void retune_fft_impl::process_tags_(const input_type *in, size_t in_count,
   std::vector<double> rx_times;
   get_tags_in_window(all_tags, 0, 0, in_count);
   get_tags(tag_, all_tags, rx_freq_tags, rx_times, in_count);
-  size_t fft_output_items = 0;
 
   if (rx_freq_tags.empty()) {
-    fft_output_items += process_items_(in_count, in, fft_output);
+    process_items_(in_count, in, fft_output);
   } else {
     for (size_t t = 0; t < rx_freq_tags.size(); ++t) {
       const auto &tag = rx_freq_tags[t];
@@ -490,7 +487,7 @@ void retune_fft_impl::process_tags_(const input_type *in, size_t in_count,
       in_first += rel;
 
       if (rel > 0) {
-        fft_output_items += process_items_(rel, in, fft_output);
+        process_items_(rel, in, fft_output);
       }
 
       const uint64_t rx_freq = (uint64_t)pmt::to_double(tag.value);
@@ -502,8 +499,6 @@ void retune_fft_impl::process_tags_(const input_type *in, size_t in_count,
       process_buckets_(rx_freq, rx_time);
     }
   }
-
-  produce(1, fft_output_items);
 }
 
 int retune_fft_impl::general_work(int noutput_items,
@@ -518,7 +513,6 @@ int retune_fft_impl::general_work(int noutput_items,
     std::copy(from, to, out);
     out_buf_.erase(from, to);
     produce(0, leftover);
-    produce(1, 0);
     return WORK_CALLED_PRODUCE;
   }
 
@@ -531,7 +525,6 @@ int retune_fft_impl::general_work(int noutput_items,
   size_t in_first = nitems_read(0);
   process_tags_(in, in_count, in_first, fft_output);
   consume_each(in_count);
-  produce(0, 0);
 
   return WORK_CALLED_PRODUCE;
 }

lib/retune_fft_impl.h

@@ -224,8 +224,8 @@ class retune_fft_impl : public retune_fft, base_impl, retuner_impl {
   void retune_now_();
   void write_items_(const input_type *in);
   void sum_items_(const input_type *in);
-  size_t process_items_(size_t c, const input_type *&in,
-                        const input_type *&fft_output);
+  void process_items_(size_t c, const input_type *&in,
+                      const input_type *&fft_output);
   void output_buckets_(const std::string &name,
                        const std::list<std::pair<double, double>> &buckets,
                        std::stringstream &ss);

python/iqtlabs/bindings/image_inference_python.cc

@@ -14,7 +14,7 @@
 /* BINDTOOL_GEN_AUTOMATIC(0)                                                       */
 /* BINDTOOL_USE_PYGCCXML(0)                                                        */
 /* BINDTOOL_HEADER_FILE(image_inference.h)                                        */
-/* BINDTOOL_HEADER_FILE_HASH(95fb94fd34e1930b2cdc80357e966443)                     */
+/* BINDTOOL_HEADER_FILE_HASH(b39ddd6fde28ff8492f4a3e4bcec592a)                     */
 /***********************************************************************************/
 
 #include <pybind11/complex.h>
@@ -55,6 +55,7 @@ void bind_image_inference(py::module& m)
              py::arg("model_server"),
              py::arg("model_name"),
              py::arg("confidence"),
+             py::arg("max_rows"),
              D(image_inference, make))
 
 

python/iqtlabs/qa_image_inference.py

@@ -286,6 +286,7 @@ def test_instance(self):
                 f"localhost:{port}",
                 model_name,
                 0.8,
+                1024,
             )
             c2r = blocks.complex_to_real(1)
             stream2vector = blocks.stream_to_vector(gr.sizeof_float, fft_size)