[VSINPU]Code improvement && Slice/Dropout OP support

cmake/onnxruntime_providers.cmake

@@ -192,32 +192,7 @@ if (onnxruntime_USE_TVM)
 endif()
 
 if (onnxruntime_USE_VSINPU)
-  add_definitions(-DUSE_VSINPU=1)
-  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-parameter")
-  file(GLOB_RECURSE onnxruntime_providers_vsinpu_srcs
-    "${ONNXRUNTIME_ROOT}/core/providers/vsinpu/builders/*.h"
-    "${ONNXRUNTIME_ROOT}/core/providers/vsinpu/builders/*.cc"
-    "${ONNXRUNTIME_ROOT}/core/providers/vsinpu/*.h"
-    "${ONNXRUNTIME_ROOT}/core/providers/vsinpu/*.cc"
-    "${ONNXRUNTIME_ROOT}/core/providers/shared/utils/utils.h"
-    "${ONNXRUNTIME_ROOT}/core/providers/shared/utils/utils.cc"
-  )
-  source_group(TREE ${ONNXRUNTIME_ROOT}/core FILES ${onnxruntime_providers_vsinpu_srcs})
-  add_library(onnxruntime_providers_vsinpu ${onnxruntime_providers_vsinpu_srcs})
-  onnxruntime_add_include_to_target(onnxruntime_providers_vsinpu
-    onnxruntime_common onnxruntime_framework onnx onnx_proto protobuf::libprotobuf-lite flatbuffers Boost::mp11
-    safeint_interface nsync::nsync_cpp)
-  add_dependencies(onnxruntime_providers_vsinpu ${onnxruntime_EXTERNAL_DEPENDENCIES})
-  set_target_properties(onnxruntime_providers_vsinpu PROPERTIES FOLDER "ONNXRuntime" LINKER_LANGUAGE CXX)
-  target_include_directories(onnxruntime_providers_vsinpu PRIVATE ${ONNXRUNTIME_ROOT} $ENV{TIM_VX_INSTALL}/include)
-
-  find_library(TIMVX_LIBRARY NAMES tim-vx PATHS $ENV{TIM_VX_INSTALL}/lib NO_DEFAULT_PATH)
-  if(TIMVX_LIBRARY)
-    target_link_libraries(onnxruntime_providers_vsinpu PRIVATE ${TIMVX_LIBRARY})
-  else()
-    message(FATAL_ERROR "Cannot find TIM-VX library!")
-  endif()
-
+  include(onnxruntime_providers_vsinpu.cmake)
 endif()
 
 if (onnxruntime_USE_XNNPACK)

cmake/onnxruntime_providers_vsinpu.cmake

@@ -0,0 +1,37 @@
+  add_definitions(-DUSE_VSINPU=1)
+  file(GLOB_RECURSE onnxruntime_providers_vsinpu_srcs
+    "${ONNXRUNTIME_ROOT}/core/providers/vsinpu/builders/*.h"
+    "${ONNXRUNTIME_ROOT}/core/providers/vsinpu/builders/*.cc"
+    "${ONNXRUNTIME_ROOT}/core/providers/vsinpu/*.h"
+    "${ONNXRUNTIME_ROOT}/core/providers/vsinpu/*.cc"
+    "${ONNXRUNTIME_ROOT}/core/providers/shared/utils/utils.h"
+    "${ONNXRUNTIME_ROOT}/core/providers/shared/utils/utils.cc"
+  )
+  source_group(TREE ${ONNXRUNTIME_ROOT}/core FILES ${onnxruntime_providers_vsinpu_srcs})
+  add_library(onnxruntime_providers_vsinpu ${onnxruntime_providers_vsinpu_srcs})
+  onnxruntime_add_include_to_target(onnxruntime_providers_vsinpu
+    onnxruntime_common onnxruntime_framework onnx onnx_proto protobuf::libprotobuf-lite flatbuffers Boost::mp11
+    safeint_interface nsync::nsync_cpp)
+  add_dependencies(onnxruntime_providers_vsinpu ${onnxruntime_EXTERNAL_DEPENDENCIES})
+  set_target_properties(onnxruntime_providers_vsinpu PROPERTIES FOLDER "ONNXRuntime" LINKER_LANGUAGE CXX)
+  target_include_directories(onnxruntime_providers_vsinpu PRIVATE ${ONNXRUNTIME_ROOT} $ENV{TIM_VX_INSTALL}/include)
+
+  find_library(TIMVX_LIBRARY NAMES tim-vx PATHS $ENV{TIM_VX_INSTALL}/lib NO_DEFAULT_PATH)
+  if(NOT TIMVX_LIBRARY)
+    message(FATAL_ERROR "TIM-VX library is not found!")
+  endif()
+
+  if(CMAKE_CROSSCOMPILING)
+    message(STATUS "VSINPU ep will be cross compiled.")
+    if(EXISTS "$ENV{VIVANTE_SDK_DIR}/drivers")
+      set(DRIVER_DIR "$ENV{VIVANTE_SDK_DIR}/drivers")
+    elseif(EXISTS "$ENV{VIVANTE_SDK_DIR}/lib")
+      set(DRIVER_DIR "$ENV{VIVANTE_SDK_DIR}/lib")
+    else()
+      message(FATAL_ERROR "Neither drivers nor lib directory exists in this VIVANTE_SDK_DIR.")
+    endif()
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-parameter -Wl,-rpath-link ${DRIVER_DIR} ${TIMVX_LIBRARY}")
+  else()
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-parameter")
+    target_link_libraries(onnxruntime_providers_vsinpu PRIVATE ${TIMVX_LIBRARY})
+  endif()

onnxruntime/core/providers/vsinpu/builders/impl/activation_op_builder.h

@@ -21,6 +21,7 @@
  *    DEALINGS IN THE SOFTWARE.
  *
  *****************************************************************************/
+#pragma once
 #include <memory>
 #include <vector>
 #include <utility>

onnxruntime/core/providers/vsinpu/builders/impl/base_op_builder.cc

@@ -100,6 +100,16 @@ bool BaseOpBuilder::HasSupportedInputOutputs(const InitializedTensorSet& initial
     }
   }
   for (const auto& output : node_unit.Outputs()) {
+    for (const auto& dim : output.node_arg.Shape()->dim()) {
+      if (!dim.has_dim_value()) {
+        LOGS_DEFAULT(WARNING) << "Dynamic shape is not supported for now, for output:" << output.node_arg.Name();
+        return false;
+      }
+      if (dim.dim_value() == 0 && output.node_arg.Shape()->dim_size() > 1) {
+        LOGS_DEFAULT(WARNING) << "Zero in shape is not supported for now, for output:" << output.node_arg.Name();
+        return false;
+      }
+    }
     if (output.quant_param.has_value()) {
       if (!has_supported_shape(output.quant_param->scale, node_unit.Name(), node_unit.OpType()))
         return false;

onnxruntime/core/providers/vsinpu/builders/impl/base_op_builder.h

@@ -40,7 +40,7 @@ class BaseOpBuilder : public IOpBuilder {
   bool IsSupported(const onnxruntime::GraphViewer& graph_viewer,
                    const NodeUnit& node_unit) const override;
   bool BuildOp(vsi::npu::GraphEP* graph_ep,
-               const onnxruntime::GraphViewer& graph_viewer, const NodeUnit& node_unit);
+               const onnxruntime::GraphViewer& graph_viewer, const NodeUnit& node_unit) override;
   virtual bool IsOpSupported(const onnxruntime::GraphViewer& graph_viewer,
                              const Node* node) const {
     return true;

onnxruntime/core/providers/vsinpu/builders/impl/cast_op_builder.h

@@ -21,6 +21,7 @@
  *    DEALINGS IN THE SOFTWARE.
  *
  *****************************************************************************/
+#pragma once
 #include <memory>
 #include <vector>
 #include <utility>

onnxruntime/core/providers/vsinpu/builders/impl/clip_op_builder.h

@@ -21,6 +21,7 @@
  *    DEALINGS IN THE SOFTWARE.
  *
  *****************************************************************************/
+#pragma once
 #include <memory>
 #include <vector>
 #include "core/providers/vsinpu/builders/impl/base_op_builder.h"

onnxruntime/core/providers/vsinpu/builders/impl/concat_op_builder.h

@@ -21,6 +21,7 @@
  *    DEALINGS IN THE SOFTWARE.
  *
  *****************************************************************************/
+#pragma once
 #include <memory>
 #include <vector>
 #include <utility>

onnxruntime/core/providers/vsinpu/builders/impl/conv_op_builder.h

@@ -21,6 +21,7 @@
  *    DEALINGS IN THE SOFTWARE.
  *
  *****************************************************************************/
+#pragma once
 #include <string>
 #include <memory>
 #include <vector>

onnxruntime/core/providers/vsinpu/builders/impl/dequantize_op_builder.h

@@ -21,6 +21,7 @@
  *    DEALINGS IN THE SOFTWARE.
  *
  *****************************************************************************/
+#pragma once
 #include <memory>
 #include <vector>
 #include <utility>

onnxruntime/core/providers/vsinpu/builders/impl/dropout_op_builder.h

@@ -0,0 +1,81 @@
+/****************************************************************************
+ *
+ *    Copyright (c) 2024 Vivante Corporation
+ *
+ *    Permission is hereby granted, free of charge, to any person obtaining a
+ *    copy of this software and associated documentation files (the "Software"),
+ *    to deal in the Software without restriction, including without limitation
+ *    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ *    and/or sell copies of the Software, and to permit persons to whom the
+ *    Software is furnished to do so, subject to the following conditions:
+ *
+ *    The above copyright notice and this permission notice shall be included in
+ *    all copies or substantial portions of the Software.
+ *
+ *    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ *    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ *    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ *    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ *    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ *    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ *    DEALINGS IN THE SOFTWARE.
+ *
+ *****************************************************************************/
+#pragma once
+#include <memory>
+#include <vector>
+#include <utility>
+#include "core/providers/vsinpu/builders/impl/base_op_builder.h"
+#include "core/providers/shared/utils/utils.h"
+
+namespace onnxruntime {
+namespace vsi {
+namespace npu {
+class DropoutOpBuilder : public BaseOpBuilder {
+  bool HasSupportedInputOutputsImpl(const InitializedTensorSet& initializers,
+                                    const NodeUnit& node_unit) const override {
+    if (node_unit.Inputs().size() > 2) {
+      const ONNX_NAMESPACE::TensorProto* tensor_proto =
+          initializers.at(node_unit.Inputs()[2].node_arg.Name());
+      std::vector<uint8_t> training_mode(1);
+      auto status = onnxruntime::utils::UnpackTensor(
+          *tensor_proto,
+          tensor_proto->has_raw_data() ? tensor_proto->raw_data().data() : nullptr,
+          tensor_proto->has_raw_data() ? tensor_proto->raw_data().size() : 0,
+          training_mode.data(), training_mode.size());
+      if (!status.IsOK()) {
+        LOGS_DEFAULT(ERROR) << "Failed to get data training mode tensor.";
+        return false;
+      }
+      if (training_mode[0] == true) {
+        LOGS_DEFAULT(WARNING) << "Only support inference typed dropout now.";
+        return false;
+      }
+    }
+    if (node_unit.Inputs().size() > 1) return false;
+    return true;
+  }
+  bool IsOpSupported(const onnxruntime::GraphViewer& graph_viewer,
+                     const Node* node) const override {
+    NodeAttrHelper helper(*node);
+    if (helper.HasAttr("seed")) {
+      LOGS_DEFAULT(WARNING) << "Not support seed in Dropout op.";
+      return false;
+    }
+    return true;
+  }
+  bool HandleBuildOp(vsi::npu::GraphEP* graph_ep,
+                     std::vector<std::shared_ptr<tim::vx::Tensor>>& inputs,
+                     std::vector<std::shared_ptr<tim::vx::Tensor>>& outputs,
+                     const NodeUnit& node_unit) override {
+    LOGS_DEFAULT(VERBOSE) << "Creating DropOut Op.";
+    auto op = graph_ep->GetGraph()->CreateOperation<tim::vx::ops::Dropout>(1.0);
+    (*op).BindInput(inputs[0]).BindOutputs(outputs);
+    graph_ep->GetOps().push_back(std::move(op));
+    return true;
+  }
+};
+}  // namespace npu
+
+}  // namespace vsi
+}  // namespace onnxruntime

onnxruntime/core/providers/vsinpu/builders/impl/elementwise_op_builder.h

@@ -22,6 +22,7 @@
  *    DEALINGS IN THE SOFTWARE.
  *
  *****************************************************************************/
+#pragma once
 #include <memory>
 #include <vector>
 #include <utility>

onnxruntime/core/providers/vsinpu/builders/impl/flatten_op_builder.h

@@ -21,6 +21,7 @@
  *    DEALINGS IN THE SOFTWARE.
  *
  *****************************************************************************/
+#pragma once
 #include <memory>
 #include <vector>
 #include <utility>

onnxruntime/core/providers/vsinpu/builders/impl/gather_op_builder.h

@@ -21,6 +21,7 @@
  *    DEALINGS IN THE SOFTWARE.
  *
  *****************************************************************************/
+#pragma once
 #include <memory>
 #include <vector>
 #include <utility>

onnxruntime/core/providers/vsinpu/builders/impl/gemm_op_builder.h

@@ -21,6 +21,7 @@
  *    DEALINGS IN THE SOFTWARE.
  *
  *****************************************************************************/
+#pragma once
 #include <memory>
 #include <vector>
 #include <utility>

onnxruntime/core/providers/vsinpu/builders/impl/matmul_op_builder.h

@@ -21,6 +21,7 @@
  *    DEALINGS IN THE SOFTWARE.
  *
  *****************************************************************************/
+#pragma once
 #include <memory>
 #include <vector>
 #include <utility>

onnxruntime/core/providers/vsinpu/builders/impl/norm_op_builder.h

@@ -21,6 +21,7 @@
  *    DEALINGS IN THE SOFTWARE.
  *
  *****************************************************************************/
+#pragma once
 #include <memory>
 #include <vector>
 #include <utility>

onnxruntime/core/providers/vsinpu/builders/impl/pool_op_builder.h

@@ -21,6 +21,7 @@
  *    DEALINGS IN THE SOFTWARE.
  *
  *****************************************************************************/
+#pragma once
 #include <memory>
 #include <vector>
 #include <utility>

onnxruntime/core/providers/vsinpu/builders/impl/qlinear_binary_op_builder.h

@@ -21,6 +21,7 @@
  *    DEALINGS IN THE SOFTWARE.
  *
  *****************************************************************************/
+#pragma once
 #include <memory>
 #include <vector>
 #include <utility>

onnxruntime/core/providers/vsinpu/builders/impl/qlinearconcat_op_builder.h

@@ -21,6 +21,7 @@
  *    DEALINGS IN THE SOFTWARE.
  *
  *****************************************************************************/
+#pragma once
 #include <memory>
 #include <vector>
 #include <utility>

onnxruntime/core/providers/vsinpu/builders/impl/qlinearconv_op_builder.h

@@ -21,6 +21,7 @@
  *    DEALINGS IN THE SOFTWARE.
  *
  *****************************************************************************/
+#pragma once
 #include <string>
 #include <memory>
 #include <vector>

onnxruntime/core/providers/vsinpu/builders/impl/qlinearmatmul_op_builder.h

@@ -21,6 +21,7 @@
  *    DEALINGS IN THE SOFTWARE.
  *
  *****************************************************************************/
+#pragma once
 #include <memory>
 #include <vector>
 #include <utility>

onnxruntime/core/providers/vsinpu/builders/impl/quantize_op_builder.h

@@ -21,6 +21,7 @@
  *    DEALINGS IN THE SOFTWARE.
  *
  *****************************************************************************/
+#pragma once
 #include <memory>
 #include <vector>
 #include <utility>

onnxruntime/core/providers/vsinpu/builders/impl/reduce_op_builder.h

@@ -21,6 +21,7 @@
  *    DEALINGS IN THE SOFTWARE.
  *
  *****************************************************************************/
+#pragma once
 #include <memory>
 #include <vector>
 #include <utility>

onnxruntime/core/providers/vsinpu/builders/impl/resize_op_builder.h

@@ -21,6 +21,7 @@
  *    DEALINGS IN THE SOFTWARE.
  *
  *****************************************************************************/
+#pragma once
 #include <memory>
 #include <vector>
 #include <utility>
@@ -136,8 +137,10 @@ class ResizeOpBuilder : public BaseOpBuilder {
         for (int i = 0; i < input_shape.size(); i++) {
           out_shape[i] = input_shape[i] * scales[input_shape.size() - 1 - i];
         }
+        target_h = static_cast<int>(out_shape[1]);
+        target_w = static_cast<int>(out_shape[0]);
         op = graph_ep->GetGraph()->CreateOperation<tim::vx::ops::Resize>(resize_type, 0, align_corners,
-                                                                         half_pixel_center, out_shape[1], out_shape[0]);
+                                                                         half_pixel_center, target_h, target_w);
       }
     }