feat: yolo v8 pose for hailo

* fix: server crash when model is none * feat: yolo v8 pose for hailo (experimental) * fix: compile errors * chore: cleanup, bug fixes
Seeed-Studio · Dec 2, 2024 · a328bdd · a328bdd
1 parent 38f4cfe
commit a328bdd
Show file tree

Hide file tree

Showing 16 changed files with 664 additions and 95 deletions.
diff --git a/sscma/core/engine/ma_engine_hailo.cpp b/sscma/core/engine/ma_engine_hailo.cpp
@@ -2,6 +2,7 @@
 
 #if MA_USE_ENGINE_HAILO
 
+#include <atomic>
 #include <chrono>
 #include <cstdint>
 #include <cstring>
@@ -20,11 +21,35 @@ ma_err_t EngineHailo::init() {
         return MA_OK;
     }
 
-    auto vdevice = VDevice::create();
-    if (!vdevice) {
+    // TODO: optimize
+    static auto get_vdevice_f = []() -> shared_ptr<VDevice> {
+        static unique_ptr<VDevice> vdevice = nullptr;
+        if (!vdevice) {
+            auto dev = VDevice::create();
+            if (!dev) {
+                return nullptr;
+            }
+            vdevice = move(dev.release());
+        }
+        if (!vdevice) {
+            return nullptr;
+        }
+        auto shared = vdevice.get();
+        auto mgr = &vdevice;
+        static atomic<size_t> ref_count = 0;
+        ref_count.fetch_add(1);
+        return shared_ptr<VDevice>(shared, [mgr](VDevice*) {
+            if (mgr) {
+                if (ref_count.fetch_sub(1) == 1) {
+                    mgr->reset();
+                }
+            }
+        });
+    };
+    _vdevice = get_vdevice_f();
+    if (!_vdevice) {
         return MA_FAILED;
     }
-    _vdevice = move(vdevice.value());
 
     return MA_OK;
 }

diff --git a/sscma/core/engine/ma_engine_hailo.h b/sscma/core/engine/ma_engine_hailo.h
@@ -52,7 +52,7 @@ class EngineHailo final : public Engine {
     ma_err_t setInput(int32_t index, const ma_tensor_t& tensor) override;
 
 private:
-    unique_ptr<VDevice> _vdevice;
+    shared_ptr<VDevice> _vdevice;
     shared_ptr<InferModel> _model;
     shared_ptr<ConfiguredInferModel> _configured_model;
     shared_ptr<ConfiguredInferModel::Bindings> _bindings;

diff --git a/sscma/core/math/ma_math.h b/sscma/core/math/ma_math.h
@@ -3,5 +3,6 @@
 
 #include "ma_math_scalars.h"
 #include "ma_math_vectors.h"
+#include "ma_math_matrix.h"
 
 #endif  // _MA_MATH_H
diff --git a/sscma/core/math/ma_math_matrix.cpp b/sscma/core/math/ma_math_matrix.cpp
@@ -0,0 +1,22 @@
+#include "ma_math_matrix.h"
+#include "ma_math_vectors.h"
+
+#include <cmath>
+
+namespace ma::math {
+
+void softmax2D(float* data, size_t rows, size_t cols) {
+    size_t size = rows * cols;
+    for (size_t i = 0; i < size; i += cols) {
+        softmax(&data[i], cols);
+    }
+}
+
+void fastSoftmax2D(float* data, size_t rows, size_t cols) {
+    size_t size = rows * cols;
+    for (size_t i = 0; i < size; i += cols) {
+        fastSoftmax(&data[i], cols);
+    }
+}
+
+}
diff --git a/sscma/core/math/ma_math_matrix.h b/sscma/core/math/ma_math_matrix.h
@@ -0,0 +1,32 @@
+#ifndef _MA_MATH_MARTRIX_H_
+#define _MA_MATH_MARTRIX_H_
+
+#include <cstddef>
+#include <cstdint>
+
+#if MA_USE_LIB_XTENSOR
+#include <xtensor/xarray.hpp>
+#include <xtensor/xmath.hpp>
+#include <xtensor/xview.hpp>
+#endif
+
+namespace ma::math {
+
+void softmax2D(float* data, size_t rows, size_t cols);
+
+void fastSoftmax2D(float* data, size_t rows, size_t cols);
+
+#if MA_USE_LIB_XTENSOR
+template <typename QT>
+static void dequantizeValues2D(xt::xarray<float>& dequantized_outputs, int index, const xt::xarray<QT>& quantized_outputs, size_t dim1, size_t dim2, float32_t qp_scale, float32_t qp_zp) {
+    for (size_t i = 0; i < dim1; i++) {
+        for (size_t j = 0; j < dim2; j++) {
+            dequantized_outputs(i, j) = (float(quantized_outputs(index, i, j)) - qp_zp) * qp_scale;
+        }
+    }
+}
+#endif
+
+}  // namespace ma::math
+
+#endif
diff --git a/sscma/core/math/ma_math_scalars.h b/sscma/core/math/ma_math_scalars.h
@@ -21,8 +21,8 @@ constexpr inline float fastLn(float x) {
         return -std::numeric_limits<float>::infinity();
     }
 
-    auto       bx{*reinterpret_cast<unsigned int*>(&x)};
-    auto       ex{bx >> 23};
+    auto bx{*reinterpret_cast<unsigned int*>(&x)};
+    auto ex{bx >> 23};
     const auto t{static_cast<signed int>(ex) - static_cast<signed int>(127)};
 
     bx = 1065353216 | (bx & 8388607);
@@ -41,17 +41,22 @@ constexpr inline float fastExp(float x) {
     const float c{8388608.f};
     const float d{2139095040.f};
 
-    if ((x < c) | (x > d)) x = (x < c) ? 0.0f : d;
+    if ((x < c) | (x > d))
+        x = (x < c) ? 0.0f : d;
 
     uint32_t n = static_cast<uint32_t>(x);
     x          = *reinterpret_cast<float*>(&n);
 
     return x;
 }
 
-constexpr inline float sigmoid(float x) { return 1.0f / (1.0f + std::exp(-x)); }
+constexpr inline float sigmoid(float x) {
+    return 1.0f / (1.0f + std::exp(-x));
+}
 
-constexpr inline float fastSigmoid(float x) { return 1.0f / (1.0f + fastExp(-x)); }
+constexpr inline float fastSigmoid(float x) {
+    return 1.0f / (1.0f + fastExp(-x));
+}
 
 constexpr inline float inverseSigmoid(float x) {
     float denominator = 1.0f - x;

diff --git a/sscma/core/model/ma_model_factory.cpp b/sscma/core/model/ma_model_factory.cpp
@@ -11,58 +11,63 @@ Model* ModelFactory::create(Engine* engine, size_t algorithm_id) {
     }
 
     switch (algorithm_id) {
-    case 0:
-    case MA_MODEL_TYPE_FOMO:
-        if (FOMO::isValid(engine)) {
-            return new FOMO(engine);
-        }
-
-    case MA_MODEL_TYPE_IMCLS:
-        if (Classifier::isValid(engine)) {
-            return new Classifier(engine);
-        }
+        case 0:
+        case MA_MODEL_TYPE_FOMO:
+            if (FOMO::isValid(engine)) {
+                return new FOMO(engine);
+            }
 
-    case MA_MODEL_TYPE_PFLD:
-        if (PFLD::isValid(engine)) {
-            return new PFLD(engine);
-        }
+        case MA_MODEL_TYPE_IMCLS:
+            if (Classifier::isValid(engine)) {
+                return new Classifier(engine);
+            }
 
-    case MA_MODEL_TYPE_YOLOV5:
-        if (YoloV5::isValid(engine)) {
-            return new YoloV5(engine);
-        }
+        case MA_MODEL_TYPE_PFLD:
+            if (PFLD::isValid(engine)) {
+                return new PFLD(engine);
+            }
 
-    case MA_MODEL_TYPE_YOLOV8_POSE:
-        if (YoloV8Pose::isValid(engine)) {
-            return new YoloV8Pose(engine);
-        }
+        case MA_MODEL_TYPE_YOLOV5:
+            if (YoloV5::isValid(engine)) {
+                return new YoloV5(engine);
+            }
 
-    case MA_MODEL_TYPE_YOLOV8:
-        if (YoloV8::isValid(engine)) {
-            return new YoloV8(engine);
-        }
+        case MA_MODEL_TYPE_YOLOV8_POSE:
+#if MA_USE_ENGINE_HAILO
+            if (YoloV8PoseHailo::isValid(engine)) {
+                return new YoloV8PoseHailo(engine);
+            }
+#endif
+            if (YoloV8Pose::isValid(engine)) {
+                return new YoloV8Pose(engine);
+            }
 
-    case MA_MODEL_TYPE_NVIDIA_DET:
-        if (NvidiaDet::isValid(engine)) {
-            return new NvidiaDet(engine);
-        }
+        case MA_MODEL_TYPE_YOLOV8:
+            if (YoloV8::isValid(engine)) {
+                return new YoloV8(engine);
+            }
 
-    case MA_MODEL_TYPE_YOLO_WORLD:
-        if (YoloWorld::isValid(engine)) {
-            return new YoloWorld(engine);
-        }
-    case MA_MODEL_TYPE_YOLO11:
-        if (Yolo11::isValid(engine)) {
-            return new Yolo11(engine);
-        }
-    case MA_MODEL_TYPE_YOLO11_POSE:
-        if (Yolo11Pose::isValid(engine)) {
-            return new Yolo11Pose(engine);
-        }
-    case MA_MODEL_TYPE_YOLO11_SEG:
-        if (Yolo11Seg::isValid(engine)) {
-            return new Yolo11Seg(engine);
-        }
+        case MA_MODEL_TYPE_NVIDIA_DET:
+            if (NvidiaDet::isValid(engine)) {
+                return new NvidiaDet(engine);
+            }
+
+        case MA_MODEL_TYPE_YOLO_WORLD:
+            if (YoloWorld::isValid(engine)) {
+                return new YoloWorld(engine);
+            }
+        case MA_MODEL_TYPE_YOLO11:
+            if (Yolo11::isValid(engine)) {
+                return new Yolo11(engine);
+            }
+        case MA_MODEL_TYPE_YOLO11_POSE:
+            if (Yolo11Pose::isValid(engine)) {
+                return new Yolo11Pose(engine);
+            }
+        case MA_MODEL_TYPE_YOLO11_SEG:
+            if (Yolo11Seg::isValid(engine)) {
+                return new Yolo11Seg(engine);
+            }
     }
 
     return nullptr;

diff --git a/sscma/core/model/ma_model_factory.h b/sscma/core/model/ma_model_factory.h
@@ -20,6 +20,7 @@
 #include "ma_model_yolov5.h"
 #include "ma_model_yolov8.h"
 #include "ma_model_yolov8_pose.h"
+#include "ma_model_yolov8_pose_hailo.h"
 
 namespace ma {
 

diff --git a/sscma/core/model/ma_model_yolov8_pose.cpp b/sscma/core/model/ma_model_yolov8_pose.cpp
@@ -24,7 +24,7 @@ static inline decltype(auto) estimateTensorHW(const ma_shape_t& shape) {
     return is_nhwc ? std::make_pair(shape.dims[1], shape.dims[2]) : std::make_pair(shape.dims[2], shape.dims[3]);
 }
 
-YoloV8Pose::YoloV8Pose(Engine* p_engine_) : PoseDetector(p_engine_, "yolo_world", MA_MODEL_TYPE_YOLO_WORLD) {
+YoloV8Pose::YoloV8Pose(Engine* p_engine_) : PoseDetector(p_engine_, "yolov8_pose", MA_MODEL_TYPE_YOLOV8_POSE) {
     MA_ASSERT(p_engine_ != nullptr);
 
     for (size_t i = 0; i < num_outputs_; ++i) {
@@ -153,7 +153,7 @@ bool YoloV8Pose::isValid(Engine* engine) {
 }
 
 const char* YoloV8Pose::getTag() {
-    return "ma::model::yolo_world";
+    return "ma::model::yolov8_pose";
 }
 
 ma_err_t YoloV8Pose::postprocess() {