From 88ee7acd3d7c700adccd59947f38f0ec4c4fba31 Mon Sep 17 00:00:00 2001 From: Rafal Sapala Date: Fri, 1 Dec 2023 17:07:57 +0100 Subject: [PATCH 1/4] Remove --- mediapipe/calculators/openvino/BUILD | 252 ------- .../openvino/internal/infer_request_wrap.hpp | 166 ----- .../openvino/openvino_converter_calculator.cc | 322 --------- .../openvino_converter_calculator.proto | 65 -- .../openvino_converter_calculator_test.cc | 169 ----- .../openvino/openvino_inference_calculator.cc | 148 ---- .../openvino_inference_calculator.proto | 60 -- .../openvino_inference_calculator_test.cc | 85 --- ...penvino_inference_calculator_test_common.h | 109 --- ...no_tensors_to_classification_calculator.cc | 200 ------ ...tensors_to_classification_calculator.proto | 42 -- ...nsors_to_classification_calculator_test.cc | 184 ----- ...envino_tensors_to_detections_calculator.cc | 465 ------------ ...ino_tensors_to_detections_calculator.proto | 75 -- .../calculators/openvino/testdata/add.bin | 0 .../calculators/openvino/testdata/add.xml | 83 --- .../openvino/testdata/labelmap.txt | 3 - mediapipe/calculators/ovms/BUILD | 144 ---- mediapipe/calculators/ovms/calculators.md | 32 - mediapipe/calculators/ovms/config.json | 9 - mediapipe/calculators/ovms/diagram.png | Bin 50805 -> 0 bytes .../calculators/ovms/modelapiovmsadapter.cc | 310 -------- .../calculators/ovms/modelapiovmsadapter.hpp | 66 -- .../ovms/openvinoinferencecalculator.cc | 670 ------------------ .../ovms/openvinoinferencecalculator.h | 44 -- .../ovms/openvinoinferencecalculator.proto | 33 - .../ovms/openvinoinferencecalculator_test.cc | 179 ----- .../openvinomodelserversessioncalculator.cc | 210 ------ .../openvinomodelserversessioncalculator.h | 51 -- ...openvinomodelserversessioncalculator.proto | 33 - ...envinomodelserversessioncalculator_test.cc | 153 ---- mediapipe/calculators/ovms/ovms_calculator.cc | 281 -------- .../calculators/ovms/ovmscalculator.proto | 49 -- .../calculators/ovms/test_data/config.json | 10 - .../ovms/test_data/dummy/1/dummy.bin | Bin 4 -> 0 bytes .../ovms/test_data/dummy/1/dummy.xml | 99 --- mediapipe/examples/desktop/hello_ovms/BUILD | 45 -- .../hello_ovms/add_two_inputs_model/1/add.bin | 0 .../hello_ovms/add_two_inputs_model/1/add.xml | 91 --- .../desktop/hello_ovms/c_api_test_utils.hpp | 75 -- .../examples/desktop/hello_ovms/config.json | 10 - .../examples/desktop/hello_ovms/hello_ovms.cc | 149 ---- ...ect_detection_desktop_openvino_graph.pbtxt | 181 ----- ...object_detection_desktop_ovms1_graph.pbtxt | 206 ------ ...object_detection_desktop_ovms2_graph.pbtxt | 247 ------- .../object_detection_desktop_ovms_graph.pbtxt | 184 ----- mediapipe/models/ovms/config_holistic.json | 46 -- setup_ovms.py | 229 ------ test_demos.sh | 4 - third_party/model_api/BUILD | 0 third_party/model_api/model_api.bzl | 64 -- 51 files changed, 6332 deletions(-) delete mode 100644 mediapipe/calculators/openvino/BUILD delete mode 100644 mediapipe/calculators/openvino/internal/infer_request_wrap.hpp delete mode 100644 mediapipe/calculators/openvino/openvino_converter_calculator.cc delete mode 100644 mediapipe/calculators/openvino/openvino_converter_calculator.proto delete mode 100644 mediapipe/calculators/openvino/openvino_converter_calculator_test.cc delete mode 100644 mediapipe/calculators/openvino/openvino_inference_calculator.cc delete mode 100644 mediapipe/calculators/openvino/openvino_inference_calculator.proto delete mode 100644 mediapipe/calculators/openvino/openvino_inference_calculator_test.cc delete mode 100644 mediapipe/calculators/openvino/openvino_inference_calculator_test_common.h delete mode 100644 mediapipe/calculators/openvino/openvino_tensors_to_classification_calculator.cc delete mode 100644 mediapipe/calculators/openvino/openvino_tensors_to_classification_calculator.proto delete mode 100644 mediapipe/calculators/openvino/openvino_tensors_to_classification_calculator_test.cc delete mode 100644 mediapipe/calculators/openvino/openvino_tensors_to_detections_calculator.cc delete mode 100644 mediapipe/calculators/openvino/openvino_tensors_to_detections_calculator.proto delete mode 100644 mediapipe/calculators/openvino/testdata/add.bin delete mode 100644 mediapipe/calculators/openvino/testdata/add.xml delete mode 100644 mediapipe/calculators/openvino/testdata/labelmap.txt delete mode 100644 mediapipe/calculators/ovms/BUILD delete mode 100644 mediapipe/calculators/ovms/calculators.md delete mode 100644 mediapipe/calculators/ovms/config.json delete mode 100644 mediapipe/calculators/ovms/diagram.png delete mode 100644 mediapipe/calculators/ovms/modelapiovmsadapter.cc delete mode 100644 mediapipe/calculators/ovms/modelapiovmsadapter.hpp delete mode 100644 mediapipe/calculators/ovms/openvinoinferencecalculator.cc delete mode 100644 mediapipe/calculators/ovms/openvinoinferencecalculator.h delete mode 100644 mediapipe/calculators/ovms/openvinoinferencecalculator.proto delete mode 100644 mediapipe/calculators/ovms/openvinoinferencecalculator_test.cc delete mode 100644 mediapipe/calculators/ovms/openvinomodelserversessioncalculator.cc delete mode 100644 mediapipe/calculators/ovms/openvinomodelserversessioncalculator.h delete mode 100644 mediapipe/calculators/ovms/openvinomodelserversessioncalculator.proto delete mode 100644 mediapipe/calculators/ovms/openvinomodelserversessioncalculator_test.cc delete mode 100644 mediapipe/calculators/ovms/ovms_calculator.cc delete mode 100644 mediapipe/calculators/ovms/ovmscalculator.proto delete mode 100644 mediapipe/calculators/ovms/test_data/config.json delete mode 100644 mediapipe/calculators/ovms/test_data/dummy/1/dummy.bin delete mode 100644 mediapipe/calculators/ovms/test_data/dummy/1/dummy.xml delete mode 100644 mediapipe/examples/desktop/hello_ovms/BUILD delete mode 100644 mediapipe/examples/desktop/hello_ovms/add_two_inputs_model/1/add.bin delete mode 100644 mediapipe/examples/desktop/hello_ovms/add_two_inputs_model/1/add.xml delete mode 100644 mediapipe/examples/desktop/hello_ovms/c_api_test_utils.hpp delete mode 100644 mediapipe/examples/desktop/hello_ovms/config.json delete mode 100644 mediapipe/examples/desktop/hello_ovms/hello_ovms.cc delete mode 100644 mediapipe/graphs/object_detection/object_detection_desktop_openvino_graph.pbtxt delete mode 100644 mediapipe/graphs/object_detection/object_detection_desktop_ovms1_graph.pbtxt delete mode 100644 mediapipe/graphs/object_detection/object_detection_desktop_ovms2_graph.pbtxt delete mode 100644 mediapipe/graphs/object_detection/object_detection_desktop_ovms_graph.pbtxt delete mode 100644 mediapipe/models/ovms/config_holistic.json delete mode 100644 setup_ovms.py delete mode 100755 test_demos.sh delete mode 100644 third_party/model_api/BUILD delete mode 100644 third_party/model_api/model_api.bzl diff --git a/mediapipe/calculators/openvino/BUILD b/mediapipe/calculators/openvino/BUILD deleted file mode 100644 index 73382b35d7..0000000000 --- a/mediapipe/calculators/openvino/BUILD +++ /dev/null @@ -1,252 +0,0 @@ -# Copyright (c) 2023 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library", "mediapipe_proto_library") - -licenses(["notice"]) - -package(default_visibility = ["//visibility:private"]) - -mediapipe_proto_library( - name = "openvino_inference_calculator_proto", - srcs = ["openvino_inference_calculator.proto"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_options_proto", - "//mediapipe/framework:calculator_proto", - ], -) - -mediapipe_proto_library( - name = "openvino_converter_calculator_proto", - srcs = ["openvino_converter_calculator.proto"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_options_proto", - "//mediapipe/framework:calculator_proto", - ], -) - -mediapipe_proto_library( - name = "openvino_tensors_to_classification_calculator_proto", - srcs = ["openvino_tensors_to_classification_calculator.proto"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_options_proto", - "//mediapipe/framework:calculator_proto", - ], -) - -mediapipe_proto_library( - name = "openvino_tensors_to_detections_calculator_proto", - srcs = ["openvino_tensors_to_detections_calculator.proto"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_options_proto", - "//mediapipe/framework:calculator_proto", - ], -) - -cc_library( - name = "openvino_converter_calculator", - srcs = ["openvino_converter_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - ":openvino_converter_calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/formats:image_frame", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/stream_handler:fixed_size_input_stream_handler", - "//mediapipe/util:resource_util", - "@linux_openvino//:openvino", - ], - alwayslink = 1, -) - -cc_library( - name = "openvino_internal_utils", - srcs = ["internal/infer_request_wrap.hpp"], - # srcs = glob([ - # "internal/*.cpp", - # "internal/*.hpp", - # ]), - visibility = ["//visibility:public"], - deps = [ - "@linux_openvino//:openvino", - ], -) - -cc_library( - name = "openvino_inference_calculator", - srcs = ["openvino_inference_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - ":openvino_internal_utils", - ":openvino_inference_calculator_cc_proto", - "//mediapipe/framework:timestamp", - "@com_google_absl//absl/base:core_headers", - "@com_google_absl//absl/memory", - "//mediapipe/framework:calculator_context", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/tool:status_util", - "@com_google_absl//absl/strings", - "@com_google_absl//absl/synchronization", - "//mediapipe/framework/deps:clock", - "//mediapipe/framework/port:status", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:map_util", - "//mediapipe/framework:packet", - "@linux_openvino//:openvino", - ] + select({ - "//conditions:default": [ - ], - }), - alwayslink = 1, -) - -cc_library( - name = "openvino_tensors_to_classification_calculator", - srcs = ["openvino_tensors_to_classification_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - ":openvino_tensors_to_classification_calculator_cc_proto", - "@com_google_absl//absl/container:node_hash_map", - "@com_google_absl//absl/strings:str_format", - "@com_google_absl//absl/types:span", - "//mediapipe/framework/formats:classification_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/formats:location", - "//mediapipe/framework/port:ret_check", - "//mediapipe/util:resource_util", - "@linux_openvino//:openvino", - ] + select({ - "//mediapipe:android": [ - "//mediapipe/util/android/file/base", - ], - "//mediapipe:ios": [ - "//mediapipe/util/android/file/base", - ], - "//mediapipe:macos": [ - "//mediapipe/framework/port:file_helpers", - ], - "//conditions:default": [ - "//mediapipe/framework/port:file_helpers", - ], - }), - alwayslink = 1, -) - -cc_library( - name = "openvino_tensors_to_detections_calculator", - srcs = ["openvino_tensors_to_detections_calculator.cc"], - copts = select({ - "//mediapipe:ios": [ - "-x objective-c++", - "-fobjc-arc", # enable reference-counting - ], - "//conditions:default": [], - }), - linkopts = select({ - "//mediapipe:ios": [ - "-framework CoreVideo", - "-framework MetalKit", - ], - "//conditions:default": [], - }), - visibility = ["//visibility:public"], - deps = [ - ":openvino_tensors_to_detections_calculator_cc_proto", - "//mediapipe/framework/formats:detection_cc_proto", - "@com_google_absl//absl/strings:str_format", - "@com_google_absl//absl/types:span", - "//mediapipe/framework/deps:file_path", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/formats:location", - "//mediapipe/framework/formats/object_detection:anchor_cc_proto", - "//mediapipe/framework/port:ret_check", - "@linux_openvino//:openvino", - ], - alwayslink = 1, -) - -# To run this with native GPU on Linux, use: -# bazel test //mediapipe/calculators/tflite:tflite_inference_calculator_test --copt=-DTFLITE_GPU_EXTRA_GLES_DEPS --copt=-DMESA_EGL_NO_X11_HEADERS --copt=-DEGL_NO_X11 --config=grte_v5 --test_strategy=local -cc_test( - name = "openvino_inference_calculator_test", - srcs = [ - "openvino_inference_calculator_test.cc", - "openvino_inference_calculator_test_common.h", - ], - data = [ - "testdata/add.bin", - "testdata/add.xml", - ], - linkstatic = 1, - deps = [ - ":openvino_inference_calculator", - ":openvino_inference_calculator_cc_proto", - "//mediapipe/calculators/core:constant_side_packet_calculator", - "//mediapipe/calculators/util:local_file_contents_calculator", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:calculator_runner", - "//mediapipe/framework/deps:file_path", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:integral_types", - "//mediapipe/framework/port:parse_text_proto", - "//mediapipe/framework/tool:validate_type", - "@com_google_absl//absl/memory", - "@com_google_absl//absl/strings", - "@com_google_absl//absl/types:optional", - ], -) - -cc_test( - name = "openvino_converter_calculator_test", - srcs = ["openvino_converter_calculator_test.cc"], - deps = [ - ":openvino_converter_calculator", - ":openvino_converter_calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:calculator_runner", - "//mediapipe/framework/formats:image_format_cc_proto", - "//mediapipe/framework/formats:image_frame", - "//mediapipe/framework/formats:image_frame_opencv", - "//mediapipe/framework/formats:matrix", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:integral_types", - "//mediapipe/framework/port:parse_text_proto", - "//mediapipe/framework/tool:validate_type", - "@com_google_absl//absl/memory", - "@com_google_absl//absl/strings", - ], -) - -cc_test( - name = "openvino_tensors_to_classification_calculator_test", - srcs = ["openvino_tensors_to_classification_calculator_test.cc"], - data = ["testdata/labelmap.txt"], - deps = [ - ":openvino_tensors_to_classification_calculator", - ":openvino_tensors_to_classification_calculator_cc_proto", - "//mediapipe/framework:calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:calculator_runner", - "//mediapipe/framework/formats:classification_cc_proto", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:parse_text_proto", - "@com_google_absl//absl/memory", - "@com_google_googletest//:gtest_main", - ], -) diff --git a/mediapipe/calculators/openvino/internal/infer_request_wrap.hpp b/mediapipe/calculators/openvino/internal/infer_request_wrap.hpp deleted file mode 100644 index 471b119f5b..0000000000 --- a/mediapipe/calculators/openvino/internal/infer_request_wrap.hpp +++ /dev/null @@ -1,166 +0,0 @@ -// Copyright (C) 2018-2022 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#pragma once - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -// clang-format off -//#include "utils.hpp" -// clang-format on - -typedef std::function - QueueCallbackFunction; - -/// @brief Wrapper class for InferenceEngine::InferRequest. Handles asynchronous callbacks and calculates execution -/// time. -class InferReqWrap final { -public: - using Ptr = std::shared_ptr; - - ~InferReqWrap() = default; - - explicit InferReqWrap(ov::CompiledModel &model, size_t id, QueueCallbackFunction callbackQueue) - : _request(model.create_infer_request()), - _id(id), - _callbackQueue(callbackQueue) { - _request.set_callback([&](const std::exception_ptr &ptr) { - _callbackQueue(_id, ptr); - }); - } - - void start_async() { - _request.start_async(); - } - - void wait() { - _request.wait(); - } - - void infer() { - _request.infer(); - _callbackQueue(_id, nullptr); - } - - void set_shape(const std::string& name, const ov::Shape& dims) { - // TODO check return status - _request.get_tensor(name).set_shape(dims); - } - - ov::Tensor get_tensor(const std::string& name) { - return _request.get_tensor(name); - } - - ov::Tensor get_output_tensor(size_t i) { - return _request.get_output_tensor(i); - } - - void set_tensor(const std::string& name, const ov::Tensor& data) { - _request.set_tensor(name, data); - } - - void set_input_tensor(size_t i, const ov::Tensor& data) { - _request.set_input_tensor(i, data); - } - -// // in case of using GPU memory we need to allocate CL buffer for -// // output blobs. By encapsulating cl buffer inside InferReqWrap -// // we will control the number of output buffers and access to it. -// std::map& get_output_cl_buffer() { -// return outputClBuffer; -// } - -private: - ov::InferRequest _request; - size_t _id; - QueueCallbackFunction _callbackQueue; -// std::map outputClBuffer; -}; - -class InferRequestsQueue final { -public: - InferRequestsQueue(ov::CompiledModel& model, size_t nireq) - { - for (size_t id = 0; id < nireq; id++) { - requests.push_back(std::make_shared(model, - id, - std::bind(&InferRequestsQueue::put_idle_request, - this, - std::placeholders::_1, - std::placeholders::_2))); - _idleIds.push(id); - } - } - - ~InferRequestsQueue() { - // Inference Request guarantee that it will wait for all asynchronous internal tasks in destructor - // So it should be released before any context that the request can use inside internal asynchronous tasks - // For example all members of InferRequestsQueue would be destroyed before `requests` vector - // So requests can try to use this members from `putIdleRequest()` that would be called from request callback - // To avoid this we should move this vector declaration after all members declaration or just clear it manually - // in destructor - requests.clear(); - } - - void put_idle_request(size_t id, - const std::exception_ptr& ptr = nullptr) { - std::unique_lock lock(_mutex); - if (ptr) { - inferenceException = ptr; - } else { - _idleIds.push(id); - } - _cv.notify_one(); - } - - InferReqWrap::Ptr get_idle_request() { - std::unique_lock lock(_mutex); - _cv.wait(lock, [this] { - if (inferenceException) { - try { - std::rethrow_exception(inferenceException); - } catch (const std::exception &ex) { - throw ex; - } - } - return _idleIds.size() > 0; - }); - auto request = requests.at(_idleIds.front()); - _idleIds.pop(); - return request; - } - - void wait_all() { - std::unique_lock lock(_mutex); - _cv.wait(lock, [this] { - if (inferenceException) { - try { - std::rethrow_exception(inferenceException); - } catch (const std::exception& ex) { - throw ex; - } - } - return _idleIds.size() == requests.size(); - }); - } - - std::vector requests; - -private: - std::queue _idleIds; - std::mutex _mutex; - std::condition_variable _cv; - std::exception_ptr inferenceException = nullptr; -}; diff --git a/mediapipe/calculators/openvino/openvino_converter_calculator.cc b/mediapipe/calculators/openvino/openvino_converter_calculator.cc deleted file mode 100644 index cb41afa4dd..0000000000 --- a/mediapipe/calculators/openvino/openvino_converter_calculator.cc +++ /dev/null @@ -1,322 +0,0 @@ -// Copyright (c) 2023 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// - -#include -#include - -#include "mediapipe/calculators/openvino/openvino_converter_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/image_frame.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/util/resource_util.h" - -#include - -namespace { -constexpr char kImageFrameTag[] = "IMAGE"; -constexpr char kGpuBufferTag[] = "IMAGE_GPU"; -constexpr char kTensorsTag[] = "TENSORS"; -constexpr char kRemoteTensorsTag[] = "TENSORS_REMOTE"; -} // namespace - -namespace mediapipe { - -// Calculator for normalizing and converting an ImageFrame or Matrix -// into a ov::Tensor or a GpuBuffer to a ov::RemoteTensor -// -// This calculator is designed to be used with the OpenVINOInferenceCalculator, -// as a pre-processing step for calculator inputs. -// -// IMAGE and IMAGE_GPU inputs are normalized to [-1,1] (default) or [0,1], -// specified by options (unless outputting a quantized tensor). -// -// Input: -// One of the following tags: -// IMAGE - ImageFrame (assumed to be 8-bit or 32-bit data). -// IMAGE_GPU - GpuBuffer (assumed to be RGBA or RGB GL texture). -// -// Output: -// One of the following tags: -// TENSORS - Vector of ov::Tensors -// TENSORS_REMOTE - Vector of ov::RemoteTensors -// -// Example use: -// node { -// calculator: "OpenVINOConverterCalculator" -// input_stream: "IMAGE:input_image" -// output_stream: "TENSORS:image_tensor" -// options: { -// [mediapipe.OpenVINOConverterCalculatorOptions.ext] { -// zero_center: true -// } -// } -// } -// -// IMPORTANT Notes: -// No conversion between CPU/GPU is done. -// Inputs/outputs must match type: CPU->CPU or GPU->GPU. -// This calculator uses FixedSizeInputStreamHandler by default. -// -// Note: Input defines output, so only these type sets are supported: -// IMAGE -> TENSORS | IMAGE_GPU -> TENSORS_GPU -// -class OpenVINOConverterCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc); - - absl::Status Open(CalculatorContext* cc) override; - absl::Status Process(CalculatorContext* cc) override; - absl::Status Close(CalculatorContext* cc) override; - - private: - absl::Status LoadOptions(CalculatorContext* cc); - template - absl::Status NormalizeImage(const ImageFrame& image_frame, - bool flip_vertically, float* tensor_ptr); - - bool initialized_ = false; - absl::optional> output_range_; - bool flip_vertically_ = false; - int max_num_channels_ = 3; -}; -REGISTER_CALCULATOR(OpenVINOConverterCalculator); - -namespace { -template -bool ShouldUseGpu(CC* cc) { -#if MEDIAPIPE_OPENVINO_GPU_SUPPORTED - return cc->Inputs().HasTag(kGpuBufferTag) || - cc->Outputs().HasTag(kTensorsGpuTag); -#else - return false; -#endif // MEDIAPIPE_OPENVINO_GPU_SUPPORTED -} -} // namespace - -absl::Status OpenVINOConverterCalculator::GetContract(CalculatorContract* cc) { - // Confirm only one of the input streams is present. - RET_CHECK(cc->Inputs().HasTag(kImageFrameTag) ^ - cc->Inputs().HasTag(kGpuBufferTag)); - - // Confirm only one of the output streams is present. - RET_CHECK(cc->Outputs().HasTag(kTensorsTag) ^ - cc->Outputs().HasTag(kRemoteTensorsTag)); - - if (cc->Inputs().HasTag(kImageFrameTag)) { - cc->Inputs().Tag(kImageFrameTag).Set(); - } -//#if !MEDIAPIPE_DISABLE_GPU -// if (cc->Inputs().HasTag(kGpuBufferTag)) { -// cc->Inputs().Tag(kGpuBufferTag).Set(); -// } -//#endif // !MEDIAPIPE_DISABLE_GPU - - if (cc->Outputs().HasTag(kTensorsTag)) { - cc->Outputs().Tag(kTensorsTag).Set>(); - } - if (cc->Outputs().HasTag(kRemoteTensorsTag)) { - cc->Outputs().Tag(kRemoteTensorsTag).Set>(); - } - - // Assign this calculator's default InputStreamHandler. - cc->SetInputStreamHandler("FixedSizeInputStreamHandler"); - - return absl::OkStatus(); -} - -absl::Status OpenVINOConverterCalculator::Open(CalculatorContext* cc) { - // Inform the framework that we always output at the same timestamp - // as we receive a packet at. - cc->SetOffset(TimestampDiff(0)); - - MP_RETURN_IF_ERROR(LoadOptions(cc)); - - return absl::OkStatus(); -} - -absl::Status OpenVINOConverterCalculator::Close(CalculatorContext* cc) { - return absl::OkStatus(); -} - -absl::Status OpenVINOConverterCalculator::Process(CalculatorContext* cc) { - RET_CHECK(cc->Inputs().HasTag(kImageFrameTag)) << "Only supporting ImageFrame inputs at the moment"; - - if (cc->Inputs().HasTag(kImageFrameTag)) { - if (cc->Inputs().Tag(kImageFrameTag).IsEmpty()) { - return absl::OkStatus(); - } - const auto &input_item = cc->Inputs().Tag(kImageFrameTag); - RET_CHECK(!input_item.IsEmpty()) << "Input cannot be empty."; - - // Extract the ImageFrame and metadata from the input packet. - const ImageFrame &image_frame = input_item.Get(); - const int bytes_per_pixel = image_frame.ByteDepth(); - - const size_t height = image_frame.Height(); - const size_t width = image_frame.Width(); - const size_t num_channels = image_frame.NumberOfChannels(); - const size_t num_components = width * height * num_channels; - ov::Shape tensor_shape({1, height, width, num_channels}); - - auto output_tensors = ::absl::make_unique>(); - - if (output_range_.has_value()) { - // Normalize - ov::Tensor tensor(ov::element::f32, tensor_shape); - float* tensor_buffer = tensor.data(); - RET_CHECK(tensor_buffer); - - if (image_frame.ByteDepth() == 1) { - MP_RETURN_IF_ERROR(NormalizeImage(image_frame, flip_vertically_, - tensor_buffer)); - } else if (image_frame.ByteDepth() == 4) { - MP_RETURN_IF_ERROR(NormalizeImage(image_frame, flip_vertically_, - tensor_buffer)); - } else { - return absl::InternalError( - "Only byte-based (8 bit) and float (32 bit) images supported."); - } - output_tensors->emplace_back(tensor); - } else { - // Copy ImageFrame data directly into OpenVINO Tensor, element type is preserved. - ov::element::Type data_type; - if (bytes_per_pixel == 1) { - data_type = ov::element::u8; - } else if (bytes_per_pixel == 2) { - data_type = ov::element::u16; - } else if (bytes_per_pixel == 4) { - data_type = ov::element::f32; - } else { - return absl::InvalidArgumentError(absl::StrCat( - "Unsupported image format (", bytes_per_pixel, " bytes per pixel)")); - } - - // Create the output tensor - // TODO: is zero-copy possible? - ov::Tensor tensor(data_type, tensor_shape); - // Copy pixel data from the ImageFrame to the tensor. - // TODO: what if there are strides in the tensor? - if (data_type == ov::element::u8) { - uint8 *dst = (uint8 *) tensor.data(); - image_frame.CopyToBuffer(dst, num_components); - } else if (data_type == ov::element::u16) { - uint16 *dst = (uint16_t *) tensor.data(); - image_frame.CopyToBuffer(dst, num_components); - } else { - float *dst = (float *) tensor.data(); - image_frame.CopyToBuffer(dst, num_components); - } - output_tensors->emplace_back(tensor); - } - - cc->Outputs() - .Tag(kTensorsTag) - .Add(output_tensors.release(), cc->InputTimestamp()); - } - // TODO: else if (kGPUBufferTag) - - return absl::OkStatus(); -} - -absl::Status OpenVINOConverterCalculator::LoadOptions(CalculatorContext* cc) { - // Get calculator options specified in the graph. - const auto& options = - cc->Options<::mediapipe::OpenVINOConverterCalculatorOptions>(); - - if (options.enable_normalization()) { - // if zero_center, set output float range to match [-1, 1] as specified in - // calculator proto. - if (options.zero_center()) { - output_range_.emplace(std::pair(-1.0, 1.0)); - } - - // Custom output_tensor_float_range values. - // If the float range is specified in pb text, use the specified values - // instead. - if (options.has_output_tensor_float_range()) { - output_range_.emplace(options.output_tensor_float_range().min(), - options.output_tensor_float_range().max()); - CHECK_GT(output_range_->second, output_range_->first); - } - - // Custom div and sub values. - if (options.use_custom_normalization()) { - output_range_.emplace(std::pair( - -options.custom_sub(), - -options.custom_sub() + 255.0 / options.custom_div())); - } - } - - // Get y-flip mode. - flip_vertically_ = options.flip_vertically(); - - // Get desired way to handle input channels. - max_num_channels_ = options.max_num_channels(); - CHECK_GE(max_num_channels_, 1); - CHECK_LE(max_num_channels_, 4); - CHECK_NE(max_num_channels_, 2); - - return absl::OkStatus(); -} - -template -absl::Status OpenVINOConverterCalculator::NormalizeImage( - const ImageFrame& image_frame, bool flip_vertically, float* tensor_ptr) { - const int height = image_frame.Height(); - const int width = image_frame.Width(); - const int channels = image_frame.NumberOfChannels(); - const int channels_preserved = std::min(channels, max_num_channels_); - const int channels_ignored = channels - channels_preserved; - - if (output_range_.has_value()) { - // If the output float range is set and we are not using custom - // normalization, normalize the pixel values from [0, 255] to the specified - // output range. - RET_CHECK_NE(output_range_->first, output_range_->second); - const float scale = (output_range_->second - output_range_->first) / 255.0f; - const float bias = output_range_->first; - - for (int i = 0; i < height; ++i) { - const T* image_ptr = reinterpret_cast( - image_frame.PixelData() + - (flip_vertically ? height - 1 - i : i) * image_frame.WidthStep()); - for (int j = 0; j < width; ++j) { - for (int c = 0; c < channels_preserved; ++c) { - *tensor_ptr++ = *image_ptr++ * scale + bias; - } - image_ptr += channels_ignored; - } - } - } else { - // [0,1], scale only (bias == 0) - // Verified that there are no precision issues with 1.0f / 255.0f expression - const float scale = 1.0f / 255.0f; - for (int i = 0; i < height; ++i) { - const T* image_ptr = reinterpret_cast( - image_frame.PixelData() + - (flip_vertically ? height - 1 - i : i) * image_frame.WidthStep()); - for (int j = 0; j < width; ++j) { - for (int c = 0; c < channels_preserved; ++c) { - *tensor_ptr++ = *image_ptr++ * scale; - } - image_ptr += channels_ignored; - } - } - } - - return absl::OkStatus(); -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/openvino/openvino_converter_calculator.proto b/mediapipe/calculators/openvino/openvino_converter_calculator.proto deleted file mode 100644 index 855076dc27..0000000000 --- a/mediapipe/calculators/openvino/openvino_converter_calculator.proto +++ /dev/null @@ -1,65 +0,0 @@ -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator.proto"; - -// Full Example: -// -// node { -// calculator: "OpenVINOConverterCalculator" -// input_stream: "IMAGE:input_image" -// output_stream: "TENSORS:image_tensor" -// options { -// [mediapipe.OpenVINOConverterCalculatorOptions.ext] { -// zero_center: true -// } -// } -// } -// -message OpenVINOConverterCalculatorOptions { - extend mediapipe.CalculatorOptions { - optional OpenVINOConverterCalculatorOptions ext = 291649831; - } - - // Normalization has to be turned on explicitly. If not, the data type of - // the ImageFrame will be untouched - optional bool enable_normalization = 1 [default = false]; - - // Choose normalization mode for output (not applied for Matrix inputs). - // true = [-1,1] - // false = [0,1] - // Ignored if normalization is disabled. - optional bool zero_center = 2 [default = true]; - - // Custom settings to override the internal scaling factors `div` and `sub`. - // Both values must be set to non-negative values. Will only take effect on - // CPU AND when |use_custom_normalization| is set to true. When these custom - // values take effect, the |zero_center| setting above will be overriden, and - // the normalized_value will be calculated as: - // normalized_value = input / custom_div - custom_sub. - optional bool use_custom_normalization = 6 [default = false]; - optional float custom_div = 7 [default = -1.0]; - optional float custom_sub = 8 [default = -1.0]; - - // Whether the input image should be flipped vertically (along the - // y-direction). This is useful, for example, when the input image is defined - // with a coordinate system where the origin is at the bottom-left corner - // (e.g., in OpenGL) whereas the ML model expects an image with a top-left - // origin. - optional bool flip_vertically = 3 [default = false]; - - // Controls how many channels of the input image get passed through to the - // tensor. Valid values are 1,3,4 only. Ignored for iOS GPU. - optional int32 max_num_channels = 4 [default = 3]; - - // Normalization option. - // Setting normalization_range results in the values normalized to - // the range [output_tensor_float_range.min, output_tensor_float_range.max]. - optional TensorFloatRange output_tensor_float_range = 5; - - message TensorFloatRange { - optional float min = 1; - optional float max = 2; - } -} diff --git a/mediapipe/calculators/openvino/openvino_converter_calculator_test.cc b/mediapipe/calculators/openvino/openvino_converter_calculator_test.cc deleted file mode 100644 index 1cac475c0c..0000000000 --- a/mediapipe/calculators/openvino/openvino_converter_calculator_test.cc +++ /dev/null @@ -1,169 +0,0 @@ -// Copyright (c) 2023 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// - -#include -#include - -#include "absl/memory/memory.h" -#include "absl/strings/substitute.h" -#include "mediapipe/calculators/openvino/openvino_converter_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/calculator_runner.h" -#include "mediapipe/framework/formats/image_format.pb.h" -#include "mediapipe/framework/formats/image_frame.h" -#include "mediapipe/framework/formats/image_frame_opencv.h" -#include "mediapipe/framework/formats/matrix.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/integral_types.h" -#include "mediapipe/framework/port/parse_text_proto.h" -#include "mediapipe/framework/port/status_matchers.h" // NOLINT -#include "mediapipe/framework/tool/validate_type.h" - -#include - -namespace mediapipe { - -using RandomEngine = std::mt19937_64; -using testing::Eq; -const uint32 kSeed = 1234; -const int kNumSizes = 8; -const int sizes[kNumSizes][2] = {{1, 1}, {12, 1}, {1, 9}, {2, 2}, - {5, 3}, {7, 13}, {16, 32}, {101, 2}}; - -class OpenVINOConverterCalculatorTest : public ::testing::Test { - protected: - std::unique_ptr graph_; -}; - -TEST_F(OpenVINOConverterCalculatorTest, CustomDivAndSub) { - CalculatorGraph graph; - // Run the calculator and verify that one output is generated. - CalculatorGraphConfig graph_config = - mediapipe::ParseTextProtoOrDie(R"pb( - input_stream: "input_image" - node { - calculator: "OpenVINOConverterCalculator" - input_stream: "IMAGE:input_image" - output_stream: "TENSORS:tensor" - options { - [mediapipe.OpenVINOConverterCalculatorOptions.ext] { - enable_normalization: true - use_custom_normalization: true - custom_div: 2.0 - custom_sub: 33.0 - } - } - } - )pb"); - std::vector output_packets; - tool::AddVectorSink("tensor", &graph_config, &output_packets); - - // Run the graph. - MP_ASSERT_OK(graph.Initialize(graph_config)); - MP_ASSERT_OK(graph.StartRun({})); - auto input_image = absl::make_unique(ImageFormat::GRAY8, 1, 1); - cv::Mat mat = mediapipe::formats::MatView(input_image.get()); - mat.at(0, 0) = 200; - MP_ASSERT_OK(graph.AddPacketToInputStream( - "input_image", Adopt(input_image.release()).At(Timestamp(0)))); - - // Wait until the calculator done processing. - MP_ASSERT_OK(graph.WaitUntilIdle()); - - // Get and process results. - const std::vector& tensor_vec = - output_packets[0].Get>(); - EXPECT_EQ(1, tensor_vec.size()); - - const ov::Tensor tensor = tensor_vec[0]; - EXPECT_EQ(ov::element::f32, tensor.get_element_type()); - EXPECT_FLOAT_EQ(67.0f, tensor.data()[0]); - - // Fully close graph at end, otherwise calculator+tensors are destroyed - // after calling WaitUntilDone(). - MP_ASSERT_OK(graph.CloseInputStream("input_image")); - MP_ASSERT_OK(graph.WaitUntilDone()); -} - -TEST_F(OpenVINOConverterCalculatorTest, SetOutputRange) { - std::vector> range_values = { - std::make_pair(0.0, 1.0), std::make_pair(-1.0, 1.0), - std::make_pair(-0.5, 0.5)}; - for (std::pair range : range_values) { - CalculatorGraph graph; - CalculatorGraphConfig graph_config = - mediapipe::ParseTextProtoOrDie( - absl::Substitute(R"( - input_stream: "input_image" - node { - calculator: "OpenVINOConverterCalculator" - input_stream: "IMAGE:input_image" - output_stream: "TENSORS:tensor" - options { - [mediapipe.OpenVINOConverterCalculatorOptions.ext] { - enable_normalization: true - output_tensor_float_range { - min: $0 - max: $1 - } - } - } - } - )", - /*$0=*/range.first, - /*$1=*/range.second)); - std::vector output_packets; - tool::AddVectorSink("tensor", &graph_config, &output_packets); - - // Run the graph. - MP_ASSERT_OK(graph.Initialize(graph_config)); - MP_ASSERT_OK(graph.StartRun({})); - auto input_image = absl::make_unique(ImageFormat::GRAY8, 1, 1); - cv::Mat mat = mediapipe::formats::MatView(input_image.get()); - mat.at(0, 0) = 200; - MP_ASSERT_OK(graph.AddPacketToInputStream( - "input_image", Adopt(input_image.release()).At(Timestamp(0)))); - - // Wait until the calculator finishes processing. - MP_ASSERT_OK(graph.WaitUntilIdle()); - EXPECT_THAT(output_packets.size(), Eq(1)); - - // Get and process results. - const std::vector& tensor_vec = - output_packets[0].Get>(); - EXPECT_THAT(tensor_vec.size(), Eq(1)); - - const ov::Tensor tensor = tensor_vec[0]; - - // Calculate the expected normalized value: - float normalized_value = - range.first + (200 * (range.second - range.first)) / 255.0; - - EXPECT_EQ(tensor.get_element_type(), ov::element::f32); - EXPECT_THAT(normalized_value, - testing::FloatNear(tensor.data()[0], - 2.0f * std::abs(tensor.data()[0]) * - std::numeric_limits::epsilon())); - - // Fully close graph at end, otherwise calculator+tensors are destroyed - // after calling WaitUntilDone(). - MP_ASSERT_OK(graph.CloseInputStream("input_image")); - MP_ASSERT_OK(graph.WaitUntilDone()); - } -} - -// TODO: add test *without* normalization - -} // namespace mediapipe diff --git a/mediapipe/calculators/openvino/openvino_inference_calculator.cc b/mediapipe/calculators/openvino/openvino_inference_calculator.cc deleted file mode 100644 index 1c5b32c8b9..0000000000 --- a/mediapipe/calculators/openvino/openvino_inference_calculator.cc +++ /dev/null @@ -1,148 +0,0 @@ -// Copyright (c) 2023 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// From benchmark_app code -// Copyright (C) 2018-2022 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#include - -#include "absl/base/thread_annotations.h" -#include "absl/memory/memory.h" -#include "absl/strings/str_split.h" -#include "absl/synchronization/mutex.h" -#include "mediapipe/calculators/openvino/openvino_inference_calculator.pb.h" -#include "mediapipe/framework/calculator_context.h" -#include "mediapipe/framework/calculator_framework.h" - -#include "mediapipe/calculators/openvino/internal/infer_request_wrap.hpp" - -namespace { - constexpr char kTensorsTag[] = "TENSORS"; - constexpr char kRemoteTensorsTag[] = "TENSORS_REMOTE"; -} // namespace - -namespace mediapipe { - -class OpenVINOInferenceCalculator : public CalculatorBase { -public: - OpenVINOInferenceCalculator() { - } - - static absl::Status GetContract(CalculatorContract *cc) { - RET_CHECK(cc->Inputs().HasTag(kTensorsTag) ^ - cc->Inputs().HasTag(kRemoteTensorsTag)); - RET_CHECK(cc->Outputs().HasTag(kTensorsTag) ^ - cc->Outputs().HasTag(kRemoteTensorsTag)); - - const auto& options = - cc->Options<::mediapipe::OpenVINOInferenceCalculatorOptions>(); - RET_CHECK(!options.model_path().empty()) -// cc->InputSidePackets().HasTag(kModelTag)) - << "Either model as side packet or model path in options is required."; - - if (cc->Inputs().HasTag(kTensorsTag)) - cc->Inputs().Tag(kTensorsTag).Set>(); - if (cc->Outputs().HasTag(kTensorsTag)) - cc->Outputs().Tag(kTensorsTag).Set>(); - - if (cc->Inputs().HasTag(kRemoteTensorsTag)) - cc->Inputs().Tag(kRemoteTensorsTag).Set>(); - if (cc->Outputs().HasTag(kRemoteTensorsTag)) - cc->Outputs().Tag(kRemoteTensorsTag).Set>(); - -// if (cc->InputSidePackets().HasTag(kModelTag)) { -// cc->InputSidePackets().Tag(kModelTag).Set(); -// } - - // Assign this calculator's default InputStreamHandler. - // TODO -// cc->SetInputStreamHandler("FixedSizeInputStreamHandler"); - - return absl::OkStatus(); - } - - absl::Status Open(CalculatorContext *cc) override { - const auto& options = - cc->Options<::mediapipe::OpenVINOInferenceCalculatorOptions>(); - - RET_CHECK(!options.model_path().empty()) - << "Model path should be defined in options"; - - // TODO: process device message - - ov::Core core; - model_ = core.compile_model(options.model_path(), "CPU"); - - // TODO: add perf hints - // TODO: get nireq from perf hints - size_t nireq = 4; - infer_requests_queue_ = std::make_unique(model_, nireq); - return absl::OkStatus(); - } - - absl::Status Process(CalculatorContext *cc) override { - // Receive tensor from input message - if (cc->Inputs().Tag(kTensorsTag).IsEmpty()) { - return absl::OkStatus(); - } - - // Get infer request - auto infer_request = infer_requests_queue_->get_idle_request(); - if (!infer_request) { - return absl::InternalError("No idle inference requests available"); - } - - // Read CPU input into tensors. - const auto& input_tensors = - cc->Inputs().Tag(kTensorsTag).Get>(); - // TODO: add support for models with >1 inputs -// RET_CHECK_GT(input_tensors.size(), 0); - RET_CHECK_EQ(input_tensors.size(), 1); - for (int i = 0; i < input_tensors.size(); ++i) { - infer_request->set_input_tensor(i, input_tensors[i]); - } -// RET_CHECK(input_tensor->data.raw); - - // TODO: use async inference - infer_request->infer(); - - // Process output tensors - auto output_tensors = absl::make_unique>(); - for (int i = 0; i < model_.outputs().size(); ++i) { - ov::Tensor out_tensor = infer_request->get_output_tensor(i); - output_tensors->emplace_back(out_tensor); - } - - // Prepare calculator output - cc->Outputs() - .Tag(kTensorsTag) - .Add(output_tensors.release(), cc->InputTimestamp()); - - return absl::OkStatus(); - } - - absl::Status Close(CalculatorContext *cc) override { - return absl::OkStatus(); - } - -private: - ov::CompiledModel model_; - std::unique_ptr infer_requests_queue_; -}; - -REGISTER_CALCULATOR(OpenVINOInferenceCalculator); - -} // namespace mediapipe diff --git a/mediapipe/calculators/openvino/openvino_inference_calculator.proto b/mediapipe/calculators/openvino/openvino_inference_calculator.proto deleted file mode 100644 index 75693dc005..0000000000 --- a/mediapipe/calculators/openvino/openvino_inference_calculator.proto +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright (c) 2023 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// - -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator.proto"; - -// Full Example: -// -// node { -// calculator: "OpenVINOInferenceCalculator" -// input_stream: "TENSOR_IN:image_tensors" -// output_stream: "TENSOR_OUT:result_tensors" -// options { -// [mediapipe.OpenVINOInferenceCalculatorOptions.ext] { -// model_path: "model.openvino" -// device { gpu {} } -// } -// } -// } -// -message OpenVINOInferenceCalculatorOptions { - extend mediapipe.CalculatorOptions { - optional OpenVINOInferenceCalculatorOptions ext = 113766539; - } - - message Device { - // Default inference provided by openvino. - message CPU {} - message GPU {} - message AUTO {} - - oneof device { - AUTO auto = 1; - CPU cpu = 2; - GPU gpu = 3; - } - } - - // Path to the OpenVINO supported model (IR, ONNX, ...) - optional string model_path = 1; - - // OpenVINO device to run inference. - optional Device device = 2; -} - diff --git a/mediapipe/calculators/openvino/openvino_inference_calculator_test.cc b/mediapipe/calculators/openvino/openvino_inference_calculator_test.cc deleted file mode 100644 index 7380e90c5e..0000000000 --- a/mediapipe/calculators/openvino/openvino_inference_calculator_test.cc +++ /dev/null @@ -1,85 +0,0 @@ -// Copyright (c) 2023 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// - -#include "absl/strings/str_replace.h" -#include "mediapipe/calculators/openvino/openvino_inference_calculator_test_common.h" -namespace mediapipe { - -// Tests a simple add model that adds two input tensors -TEST(OpenVINOInferenceCalculatorTest, SmokeTest) { - std::string graph_proto = R"( - input_stream: "tensor_in" - node { - calculator: "OpenVINOInferenceCalculator" - input_stream: "TENSORS:tensor_in" - output_stream: "TENSORS:tensor_out" - options { - [mediapipe.OpenVINOInferenceCalculatorOptions.ext] { - model_path: "mediapipe/calculators/openvino/testdata/add.xml" - $device - } - } - } - )"; - // Test CPU inference only. - DoSmokeTest(/*graph_proto=*/absl::StrReplaceAll( - graph_proto, {{"$device", "device { cpu {} }"}})); -// DoSmokeTest(/*graph_proto=*/absl::StrReplaceAll( -// graph_proto, {{"$device", "device { cpu {} }"}})); -} - -// TEST(OpenVINOInferenceCalculatorTest, SmokeTest_ModelAsInputSidePacket) { -// std::string graph_proto = R"( -// input_stream: "tensor_in" -// -// node { -// calculator: "ConstantSidePacketCalculator" -// output_side_packet: "PACKET:model_path" -// options: { -// [mediapipe.ConstantSidePacketCalculatorOptions.ext]: { -// packet { string_value: "mediapipe/calculators/openvino/testdata/add.bin" } -// } -// } -// } -// -// node { -// calculator: "LocalFileContentsCalculator" -// input_side_packet: "FILE_PATH:model_path" -// output_side_packet: "CONTENTS:model_blob" -// } -// -// node { -// calculator: "OpenVINOModelCalculator" -// input_side_packet: "MODEL_BLOB:model_blob" -// output_side_packet: "MODEL:model" -// } -// -// node { -// calculator: "OpenVINOInferenceCalculator" -// input_stream: "TENSORS:tensor_in" -// output_stream: "TENSORS:tensor_out" -// input_side_packet: "MODEL:model" -// options { -// [mediapipe.OpenVINOInferenceCalculatorOptions.ext] { -// use_gpu: false -// device { openvino {} } -// } -// } -// } -// )"; -// DoSmokeTest(graph_proto); -// } - -} // namespace mediapipe diff --git a/mediapipe/calculators/openvino/openvino_inference_calculator_test_common.h b/mediapipe/calculators/openvino/openvino_inference_calculator_test_common.h deleted file mode 100644 index dc7fcf964b..0000000000 --- a/mediapipe/calculators/openvino/openvino_inference_calculator_test_common.h +++ /dev/null @@ -1,109 +0,0 @@ -// Copyright (c) 2023 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// - -#ifndef MEDIAPIPE_CALCULATORS_OPENVINO_OPENVINO_INFERENCE_CALCULATOR_TEST_H_ -#define MEDIAPIPE_CALCULATORS_OPENVINO_OPENVINO_INFERENCE_CALCULATOR_TEST_H_ - -#include -#include -#include - -#include "absl/strings/str_replace.h" -#include "absl/strings/string_view.h" -#include "mediapipe/calculators/openvino/openvino_inference_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/calculator_runner.h" -#include "mediapipe/framework/deps/file_path.h" -#include "mediapipe/framework/port/gmock.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/integral_types.h" -#include "mediapipe/framework/port/parse_text_proto.h" -#include "mediapipe/framework/port/status_matchers.h" // NOLINT -#include "mediapipe/framework/tool/validate_type.h" - -#include - -#ifdef __APPLE__ -#include -#endif // defined(__APPLE__) - -namespace mediapipe { - -template -void DoSmokeTest(const std::string& graph_proto) { - const int width = 8; - const int height = 8; - const int channels = 3; - - static_assert(std::is_same_v, - "Only uint8 currently supported."); - - // Prepare interpreter and input tensor. - ov::Tensor input_tensor(ov::element::u8, {1, channels, width, height}); - - T* input_tensor_buffer = input_tensor.data(); - ASSERT_NE(input_tensor_buffer, nullptr); - for (int i = 0; i < width * height * channels; i++) { - input_tensor_buffer[i] = 2; - } - - auto input_vec = absl::make_unique>(); - input_vec->emplace_back(input_tensor); - - auto input_vec2 = absl::make_unique>(); - input_vec2->emplace_back(input_tensor); - - // Prepare single calculator graph to and wait for packets. - CalculatorGraphConfig graph_config = - ParseTextProtoOrDie(graph_proto); - std::vector output_packets; - tool::AddVectorSink("tensor_out", &graph_config, &output_packets); - CalculatorGraph graph(graph_config); - MP_ASSERT_OK(graph.StartRun({})); - - // Push the tensor into the graph. - MP_ASSERT_OK(graph.AddPacketToInputStream( - "tensor_in", Adopt(input_vec.release()).At(Timestamp(0)))); - // Wait until the calculator done processing. - MP_ASSERT_OK(graph.WaitUntilIdle()); - ASSERT_EQ(1, output_packets.size()); - - // Get and process results - the model - const std::vector& result_vec = - output_packets[0].Get>(); - ASSERT_EQ(1, result_vec.size()); - - const ov::Tensor& out_tensor = result_vec[0]; - const ov::Shape& out_shape = out_tensor.get_shape(); - ASSERT_EQ(out_shape[0], 1); - ASSERT_EQ(out_shape[1], channels * 2); - ASSERT_EQ(out_shape[2], width); - ASSERT_EQ(out_shape[3], height); - - const T* result_buffer = out_tensor.data(); - ASSERT_NE(result_buffer, nullptr); - for (int i = 0; i < 2 * width * height * channels; i++) { - ASSERT_EQ(4, result_buffer[i]); - } - - // Fully close graph at end, otherwise calculator+tensors are destroyed - // after calling WaitUntilDone(). - MP_ASSERT_OK(graph.CloseInputStream("tensor_in")); - MP_ASSERT_OK(graph.WaitUntilDone()); -} - -} // namespace mediapipe - -#endif // MEDIAPIPE_CALCULATORS_OPENVINO_OPENVINO_INFERENCE_CALCULATOR_TEST_H_ diff --git a/mediapipe/calculators/openvino/openvino_tensors_to_classification_calculator.cc b/mediapipe/calculators/openvino/openvino_tensors_to_classification_calculator.cc deleted file mode 100644 index 26313858e5..0000000000 --- a/mediapipe/calculators/openvino/openvino_tensors_to_classification_calculator.cc +++ /dev/null @@ -1,200 +0,0 @@ -// Copyright (c) 2023 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// - -#include -#include -#include - -#include "absl/container/node_hash_map.h" -#include "absl/strings/str_format.h" -#include "absl/types/span.h" -#include "mediapipe/calculators/openvino/openvino_tensors_to_classification_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/classification.pb.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/util/resource_util.h" -#if defined(MEDIAPIPE_MOBILE) -#include "mediapipe/util/android/file/base/file.h" -#include "mediapipe/util/android/file/base/helpers.h" -#else -#include "mediapipe/framework/port/file_helpers.h" -#endif -#include - -namespace mediapipe { - -// Convert result OpenVINO tensors from classification models into MediaPipe -// classifications. -// -// Input: -// TENSORS - Vector of ov::Tensor of type float32 containing one -// tensor, the size of which must be (1, * num_classes). -// Output: -// CLASSIFICATIONS - Result MediaPipe ClassificationList. The score and index -// fields of each classification are set, while the label -// field is only set if label_map_path is provided. -// -// Usage example: -// node { -// calculator: "OpenVINOTensorToClassificationCalculator" -// input_stream: "TENSORS:tensors" -// output_stream: "CLASSIFICATIONS:classifications" -// options: { -// [mediapipe.OpenVINOTensorsToClassificationCalculatorOptions.ext] { -// num_classes: 1024 -// min_score_threshold: 0.1 -// label_map_path: "labelmap.txt" -// } -// } -// } -class OpenVINOTensorsToClassificationCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc); - - absl::Status Open(CalculatorContext* cc) override; - absl::Status Process(CalculatorContext* cc) override; - absl::Status Close(CalculatorContext* cc) override; - - private: - ::mediapipe::OpenVINOTensorsToClassificationCalculatorOptions options_; - int top_k_ = 0; - absl::node_hash_map label_map_; - bool label_map_loaded_ = false; -}; -REGISTER_CALCULATOR(OpenVINOTensorsToClassificationCalculator); - -absl::Status OpenVINOTensorsToClassificationCalculator::GetContract( - CalculatorContract* cc) { - RET_CHECK(!cc->Inputs().GetTags().empty()); - RET_CHECK(!cc->Outputs().GetTags().empty()); - - if (cc->Inputs().HasTag("TENSORS")) { - cc->Inputs().Tag("TENSORS").Set>(); - } - - if (cc->Outputs().HasTag("CLASSIFICATIONS")) { - cc->Outputs().Tag("CLASSIFICATIONS").Set(); - } - - return absl::OkStatus(); -} - -absl::Status OpenVINOTensorsToClassificationCalculator::Open( - CalculatorContext* cc) { - cc->SetOffset(TimestampDiff(0)); - - options_ = cc->Options< - ::mediapipe::OpenVINOTensorsToClassificationCalculatorOptions>(); - - top_k_ = options_.top_k(); - if (options_.has_label_map_path()) { - std::string string_path; - ASSIGN_OR_RETURN(string_path, - PathToResourceAsFile(options_.label_map_path())); - std::string label_map_string; - MP_RETURN_IF_ERROR(file::GetContents(string_path, &label_map_string)); - - std::istringstream stream(label_map_string); - std::string line; - int i = 0; - while (std::getline(stream, line)) { - label_map_[i++] = line; - } - label_map_loaded_ = true; - } - - return absl::OkStatus(); -} - -absl::Status OpenVINOTensorsToClassificationCalculator::Process( - CalculatorContext* cc) { - const auto& input_tensors = - cc->Inputs().Tag("TENSORS").Get>(); - - RET_CHECK_EQ(input_tensors.size(), 1); - - const ov::Tensor& raw_score_tensor = input_tensors[0]; - RET_CHECK_EQ(raw_score_tensor.get_shape().size(), 2) << "Expecting tensor with 2 dimensions for classification results."; - int num_classes = raw_score_tensor.get_shape()[1]; - - if (options_.binary_classification()) { - RET_CHECK_EQ(num_classes, 1); - // Number of classes for binary classification. - num_classes = 2; - } - if (label_map_loaded_) { - RET_CHECK_EQ(num_classes, label_map_.size()); - } - const float* raw_scores = raw_score_tensor.data(); - - auto classification_list = absl::make_unique(); - if (options_.binary_classification()) { - Classification* class_first = classification_list->add_classification(); - Classification* class_second = classification_list->add_classification(); - class_first->set_index(0); - class_second->set_index(1); - class_first->set_score(raw_scores[0]); - class_second->set_score(1. - raw_scores[0]); - - if (label_map_loaded_) { - class_first->set_label(label_map_[0]); - class_second->set_label(label_map_[1]); - } - } else { - for (int i = 0; i < num_classes; ++i) { - if (options_.has_min_score_threshold() && - raw_scores[i] < options_.min_score_threshold()) { - continue; - } - Classification* classification = - classification_list->add_classification(); - classification->set_index(i); - classification->set_score(raw_scores[i]); - - if (label_map_loaded_) { - classification->set_label(label_map_[i]); - } - } - } - - // Note that partial_sort will raise error when top_k_ > - // classification_list->classification_size(). - CHECK_GE(classification_list->classification_size(), top_k_); - auto raw_classification_list = classification_list->mutable_classification(); - if (top_k_ > 0 && classification_list->classification_size() >= top_k_) { - std::partial_sort(raw_classification_list->begin(), - raw_classification_list->begin() + top_k_, - raw_classification_list->end(), - [](const Classification a, const Classification b) { - return a.score() > b.score(); - }); - - // Resizes the underlying list to have only top_k_ classifications. - raw_classification_list->DeleteSubrange( - top_k_, raw_classification_list->size() - top_k_); - } - cc->Outputs() - .Tag("CLASSIFICATIONS") - .Add(classification_list.release(), cc->InputTimestamp()); - - return absl::OkStatus(); -} - -absl::Status OpenVINOTensorsToClassificationCalculator::Close( - CalculatorContext* cc) { - return absl::OkStatus(); -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/openvino/openvino_tensors_to_classification_calculator.proto b/mediapipe/calculators/openvino/openvino_tensors_to_classification_calculator.proto deleted file mode 100644 index d4e318bfe2..0000000000 --- a/mediapipe/calculators/openvino/openvino_tensors_to_classification_calculator.proto +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright (c) 2023 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// - -// The option proto for the OpenVINOTensorsToClassificationCalculator. - -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator.proto"; - -message OpenVINOTensorsToClassificationCalculatorOptions { - extend .mediapipe.CalculatorOptions { - optional OpenVINOTensorsToClassificationCalculatorOptions ext = 211376654; - } - - // Score threshold for perserving the class. - optional float min_score_threshold = 1; - // Number of highest scoring labels to output. If top_k is not positive then - // all labels are used. - optional int32 top_k = 2; - // Path to a label map file for getting the actual name of class ids. - optional string label_map_path = 3; - // Whether the input is a single float for binary classification. - // When true, only a single float is expected in the input tensor and the - // label map, if provided, is expected to have exactly two labels. - // The single score(float) represent the probability of first label, and - // 1 - score is the probabilility of the second label. - optional bool binary_classification = 4; -} diff --git a/mediapipe/calculators/openvino/openvino_tensors_to_classification_calculator_test.cc b/mediapipe/calculators/openvino/openvino_tensors_to_classification_calculator_test.cc deleted file mode 100644 index 301156734b..0000000000 --- a/mediapipe/calculators/openvino/openvino_tensors_to_classification_calculator_test.cc +++ /dev/null @@ -1,184 +0,0 @@ -// Copyright (c) 2023 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// - -#include - -#include "absl/memory/memory.h" -#include "mediapipe/calculators/openvino/openvino_tensors_to_classification_calculator.pb.h" -#include "mediapipe/framework/calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/calculator_runner.h" -#include "mediapipe/framework/formats/classification.pb.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/parse_text_proto.h" -#include "mediapipe/framework/port/status_matchers.h" - -#include - -namespace mediapipe { - -using mediapipe::ParseTextProtoOrDie; -using Node = ::mediapipe::CalculatorGraphConfig::Node; - -class OpenVINOTensorsToClassificationCalculatorTest : public ::testing::Test { - protected: - void BuildGraph(mediapipe::CalculatorRunner* runner, - const std::vector& scores) { - - std::vector dims(2); - dims[0] = 1; - dims[1] = scores.size(); - - ov::Tensor tensor(ov::element::f32, dims); - - float* tensor_buffer = tensor.data(); - ASSERT_NE(tensor_buffer, nullptr); - for (int i = 0; i < scores.size(); ++i) { - tensor_buffer[i] = scores[i]; - } - - auto tensors = absl::make_unique>(); - tensors->emplace_back(tensor); - - int64 stream_timestamp = 0; - auto& input_stream_packets = - runner->MutableInputs()->Tag("TENSORS").packets; - - input_stream_packets.push_back( - mediapipe::Adopt(tensors.release()) - .At(mediapipe::Timestamp(stream_timestamp++))); - } -}; - -TEST_F(OpenVINOTensorsToClassificationCalculatorTest, CorrectOutput) { - mediapipe::CalculatorRunner runner(ParseTextProtoOrDie(R"pb( - calculator: "OpenVINOTensorsToClassificationCalculator" - input_stream: "TENSORS:tensors" - output_stream: "CLASSIFICATIONS:classifications" - options { - [mediapipe.OpenVINOTensorsToClassificationCalculatorOptions.ext] {} - } - )pb")); - - BuildGraph(&runner, {0, 0.5, 1}); - MP_ASSERT_OK(runner.Run()); - - const auto& output_packets_ = runner.Outputs().Tag("CLASSIFICATIONS").packets; - - EXPECT_EQ(1, output_packets_.size()); - - const auto& classification_list = - output_packets_[0].Get(); - EXPECT_EQ(3, classification_list.classification_size()); - - // Verify that the label_id and score fields are set correctly. - for (int i = 0; i < classification_list.classification_size(); ++i) { - EXPECT_EQ(i, classification_list.classification(i).index()); - EXPECT_EQ(i * 0.5, classification_list.classification(i).score()); - ASSERT_FALSE(classification_list.classification(i).has_label()); - } -} - -TEST_F(OpenVINOTensorsToClassificationCalculatorTest, - CorrectOutputWithLabelMapPath) { - mediapipe::CalculatorRunner runner(ParseTextProtoOrDie(R"pb( - calculator: "OpenVINOTensorsToClassificationCalculator" - input_stream: "TENSORS:tensors" - output_stream: "CLASSIFICATIONS:classifications" - options { - [mediapipe.OpenVINOTensorsToClassificationCalculatorOptions.ext] { - label_map_path: "mediapipe/calculators/openvino/testdata/labelmap.txt" - } - } - )pb")); - - BuildGraph(&runner, {0, 0.5, 1}); - MP_ASSERT_OK(runner.Run()); - - const auto& output_packets_ = runner.Outputs().Tag("CLASSIFICATIONS").packets; - - EXPECT_EQ(1, output_packets_.size()); - - const auto& classification_list = - output_packets_[0].Get(); - EXPECT_EQ(3, classification_list.classification_size()); - - // Verify that the label field is set. - for (int i = 0; i < classification_list.classification_size(); ++i) { - EXPECT_EQ(i, classification_list.classification(i).index()); - EXPECT_EQ(i * 0.5, classification_list.classification(i).score()); - ASSERT_TRUE(classification_list.classification(i).has_label()); - } -} - -TEST_F(OpenVINOTensorsToClassificationCalculatorTest, - CorrectOutputWithLabelMinScoreThreshold) { - mediapipe::CalculatorRunner runner(ParseTextProtoOrDie(R"pb( - calculator: "OpenVINOTensorsToClassificationCalculator" - input_stream: "TENSORS:tensors" - output_stream: "CLASSIFICATIONS:classifications" - options { - [mediapipe.OpenVINOTensorsToClassificationCalculatorOptions.ext] { - min_score_threshold: 0.6 - } - } - )pb")); - - BuildGraph(&runner, {0, 0.5, 1}); - MP_ASSERT_OK(runner.Run()); - - const auto& output_packets_ = runner.Outputs().Tag("CLASSIFICATIONS").packets; - - EXPECT_EQ(1, output_packets_.size()); - - const auto& classification_list = - output_packets_[0].Get(); - - // Verify that the low score labels are filtered out. - EXPECT_EQ(1, classification_list.classification_size()); - EXPECT_EQ(1, classification_list.classification(0).score()); -} - -TEST_F(OpenVINOTensorsToClassificationCalculatorTest, CorrectOutputWithTopK) { - mediapipe::CalculatorRunner runner(ParseTextProtoOrDie(R"pb( - calculator: "OpenVINOTensorsToClassificationCalculator" - input_stream: "TENSORS:tensors" - output_stream: "CLASSIFICATIONS:classifications" - options { - [mediapipe.OpenVINOTensorsToClassificationCalculatorOptions.ext] { - top_k: 2 - } - } - )pb")); - - BuildGraph(&runner, {0, 0.5, 1}); - MP_ASSERT_OK(runner.Run()); - - const auto& output_packets_ = runner.Outputs().Tag("CLASSIFICATIONS").packets; - - EXPECT_EQ(1, output_packets_.size()); - - const auto& classification_list = - output_packets_[0].Get(); - - // Verify that the only top2 labels are left. - EXPECT_EQ(2, classification_list.classification_size()); - for (int i = 0; i < classification_list.classification_size(); ++i) { - EXPECT_EQ((classification_list.classification_size() - i) * 0.5, - classification_list.classification(i).score()); - } -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/openvino/openvino_tensors_to_detections_calculator.cc b/mediapipe/calculators/openvino/openvino_tensors_to_detections_calculator.cc deleted file mode 100644 index 03946d2ba5..0000000000 --- a/mediapipe/calculators/openvino/openvino_tensors_to_detections_calculator.cc +++ /dev/null @@ -1,465 +0,0 @@ -// Copyright (c) 2023 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// - -#include -#include - -#include "absl/strings/str_format.h" -#include "absl/types/span.h" -#include "mediapipe/calculators/openvino/openvino_tensors_to_detections_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/deps/file_path.h" -#include "mediapipe/framework/formats/detection.pb.h" -#include "mediapipe/framework/formats/location.h" -#include "mediapipe/framework/formats/object_detection/anchor.pb.h" -#include "mediapipe/framework/port/ret_check.h" - -#include - -namespace { -constexpr int kNumInputTensorsWithAnchors = 3; -constexpr int kNumCoordsPerBox = 4; - -constexpr char kTensorsTag[] = "TENSORS"; -} // namespace - -namespace mediapipe { - -namespace { - -void ConvertRawValuesToAnchors(const float* raw_anchors, int num_boxes, - std::vector* anchors) { - anchors->clear(); - for (int i = 0; i < num_boxes; ++i) { - Anchor new_anchor; - new_anchor.set_y_center(raw_anchors[i * kNumCoordsPerBox + 0]); - new_anchor.set_x_center(raw_anchors[i * kNumCoordsPerBox + 1]); - new_anchor.set_h(raw_anchors[i * kNumCoordsPerBox + 2]); - new_anchor.set_w(raw_anchors[i * kNumCoordsPerBox + 3]); - anchors->push_back(new_anchor); - } -} - -void ConvertAnchorsToRawValues(const std::vector& anchors, - int num_boxes, float* raw_anchors) { - CHECK_EQ(anchors.size(), num_boxes); - int box = 0; - for (const auto& anchor : anchors) { - raw_anchors[box * kNumCoordsPerBox + 0] = anchor.y_center(); - raw_anchors[box * kNumCoordsPerBox + 1] = anchor.x_center(); - raw_anchors[box * kNumCoordsPerBox + 2] = anchor.h(); - raw_anchors[box * kNumCoordsPerBox + 3] = anchor.w(); - ++box; - } -} - -} // namespace - -// Convert result TFLite tensors from object detection models into MediaPipe -// Detections. -// -// Input: -// TENSORS - Vector of ov::Tensorof type kOpenVINOFloat32. The vector of -// tensors can have 2 or 3 tensors. First tensor is the predicted -// raw boxes/keypoints. The size of the values must be (num_boxes -// * num_predicted_values). Second tensor is the score tensor. The -// size of the valuse must be (num_boxes * num_classes). It's -// optional to pass in a third tensor for anchors (e.g. for SSD -// models) depend on the outputs of the detection model. The size -// of anchor tensor must be (num_boxes * 4). -// TENSORS_GPU - vector of GlBuffer of MTLBuffer. -// Output: -// DETECTIONS - Result MediaPipe detections. -// -// Usage example: -// node { -// calculator: "OpenVINOTensorsToDetectionsCalculator" -// input_stream: "TENSORS:tensors" -// input_side_packet: "ANCHORS:anchors" -// output_stream: "DETECTIONS:detections" -// options: { -// [mediapipe.OpenVINOTensorsToDetectionsCalculatorOptions.ext] { -// num_classes: 91 -// num_boxes: 1917 -// num_coords: 4 -// ignore_classes: [0, 1, 2] -// x_scale: 10.0 -// y_scale: 10.0 -// h_scale: 5.0 -// w_scale: 5.0 -// } -// } -// } -class OpenVINOTensorsToDetectionsCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc); - - absl::Status Open(CalculatorContext* cc) override; - absl::Status Process(CalculatorContext* cc) override; - absl::Status Close(CalculatorContext* cc) override; - - private: - absl::Status ProcessCPU(CalculatorContext* cc, - std::vector* output_detections); - - absl::Status LoadOptions(CalculatorContext* cc); - absl::Status DecodeBoxes(const float* raw_boxes, - const std::vector& anchors, - std::vector* boxes); - absl::Status ConvertToDetections(const float* detection_boxes, - const float* detection_scores, - const int* detection_classes, - std::vector* output_detections); - Detection ConvertToDetection(float box_ymin, float box_xmin, float box_ymax, - float box_xmax, float score, int class_id, - bool flip_vertically); - - int num_classes_ = 0; - int num_boxes_ = 0; - int num_coords_ = 0; - std::set ignore_classes_; - - ::mediapipe::OpenVINOTensorsToDetectionsCalculatorOptions options_; - std::vector anchors_; - bool side_packet_anchors_{}; - - bool anchors_init_ = false; -}; -REGISTER_CALCULATOR(OpenVINOTensorsToDetectionsCalculator); - -absl::Status OpenVINOTensorsToDetectionsCalculator::GetContract( - CalculatorContract* cc) { - RET_CHECK(!cc->Inputs().GetTags().empty()); - RET_CHECK(!cc->Outputs().GetTags().empty()); - - if (cc->Inputs().HasTag(kTensorsTag)) { - cc->Inputs().Tag(kTensorsTag).Set>(); - } - - if (cc->Outputs().HasTag("DETECTIONS")) { - cc->Outputs().Tag("DETECTIONS").Set>(); - } - - if (cc->InputSidePackets().UsesTags()) { - if (cc->InputSidePackets().HasTag("ANCHORS")) { - cc->InputSidePackets().Tag("ANCHORS").Set>(); - } - } - - return absl::OkStatus(); -} - -absl::Status OpenVINOTensorsToDetectionsCalculator::Open(CalculatorContext* cc) { - cc->SetOffset(TimestampDiff(0)); - - MP_RETURN_IF_ERROR(LoadOptions(cc)); - side_packet_anchors_ = cc->InputSidePackets().HasTag("ANCHORS"); - - return absl::OkStatus(); -} - -absl::Status OpenVINOTensorsToDetectionsCalculator::Process( - CalculatorContext* cc) { - if (cc->Inputs().Tag(kTensorsTag).IsEmpty()) { - return absl::OkStatus(); - } - - auto output_detections = absl::make_unique>(); - - MP_RETURN_IF_ERROR(ProcessCPU(cc, output_detections.get())); - - // Output - if (cc->Outputs().HasTag("DETECTIONS")) { - cc->Outputs() - .Tag("DETECTIONS") - .Add(output_detections.release(), cc->InputTimestamp()); - } - - return absl::OkStatus(); -} - -absl::Status OpenVINOTensorsToDetectionsCalculator::ProcessCPU( - CalculatorContext* cc, std::vector* output_detections) { - const auto& input_tensors = - cc->Inputs().Tag(kTensorsTag).Get>(); - - if (input_tensors.size() == 2 || - input_tensors.size() == kNumInputTensorsWithAnchors) { - // Postprocessing on CPU for model without postprocessing op. E.g. output - // raw score tensor and box tensor. Anchor decoding will be handled below. - const ov::Tensor* raw_box_tensor = &input_tensors[0]; - const ov::Tensor* raw_score_tensor = &input_tensors[1]; - - // TODO: Add flexible input tensor size handling. - CHECK_EQ(raw_box_tensor->get_shape().size(), 3); - CHECK_EQ(raw_box_tensor->get_shape()[0], 1); - CHECK_EQ(raw_box_tensor->get_shape()[1], num_boxes_); - CHECK_EQ(raw_box_tensor->get_shape()[2], num_coords_); - CHECK_EQ(raw_score_tensor->get_shape().size(), 3); - CHECK_EQ(raw_score_tensor->get_shape()[0], 1); - CHECK_EQ(raw_score_tensor->get_shape()[1], num_boxes_); - CHECK_EQ(raw_score_tensor->get_shape()[2], num_classes_); - const float* raw_boxes = raw_box_tensor->data(); - const float* raw_scores = raw_score_tensor->data(); - - // TODO: Support other options to load anchors. - if (!anchors_init_) { - if (input_tensors.size() == kNumInputTensorsWithAnchors) { - const ov::Tensor* anchor_tensor = &input_tensors[2]; - CHECK_EQ(anchor_tensor->get_shape().size(), 2); - CHECK_EQ(anchor_tensor->get_shape()[0], num_boxes_); - CHECK_EQ(anchor_tensor->get_shape()[1], kNumCoordsPerBox); - const float* raw_anchors = anchor_tensor->data(); - ConvertRawValuesToAnchors(raw_anchors, num_boxes_, &anchors_); - } else if (side_packet_anchors_) { - CHECK(!cc->InputSidePackets().Tag("ANCHORS").IsEmpty()); - anchors_ = - cc->InputSidePackets().Tag("ANCHORS").Get>(); - } else { - return absl::UnavailableError("No anchor data available."); - } - anchors_init_ = true; - } - std::vector boxes(num_boxes_ * num_coords_); - MP_RETURN_IF_ERROR(DecodeBoxes(raw_boxes, anchors_, &boxes)); - - std::vector detection_scores(num_boxes_); - std::vector detection_classes(num_boxes_); - - // Filter classes by scores. - for (int i = 0; i < num_boxes_; ++i) { - int class_id = -1; - float max_score = -std::numeric_limits::max(); - // Find the top score for box i. - for (int score_idx = 0; score_idx < num_classes_; ++score_idx) { - if (ignore_classes_.find(score_idx) == ignore_classes_.end()) { - auto score = raw_scores[i * num_classes_ + score_idx]; - if (options_.sigmoid_score()) { - if (options_.has_score_clipping_thresh()) { - score = score < -options_.score_clipping_thresh() - ? -options_.score_clipping_thresh() - : score; - score = score > options_.score_clipping_thresh() - ? options_.score_clipping_thresh() - : score; - } - score = 1.0f / (1.0f + std::exp(-score)); - } - if (max_score < score) { - max_score = score; - class_id = score_idx; - } - } - } - detection_scores[i] = max_score; - detection_classes[i] = class_id; - } - - MP_RETURN_IF_ERROR( - ConvertToDetections(boxes.data(), detection_scores.data(), - detection_classes.data(), output_detections)); - } else { - // Postprocessing on CPU with postprocessing op (e.g. anchor decoding and - // non-maximum suppression) within the model. - RET_CHECK_EQ(input_tensors.size(), 4); - - const ov::Tensor* detection_boxes_tensor = &input_tensors[0]; - const ov::Tensor* detection_classes_tensor = &input_tensors[1]; - const ov::Tensor* detection_scores_tensor = &input_tensors[2]; - const ov::Tensor* num_boxes_tensor = &input_tensors[3]; - RET_CHECK_EQ(num_boxes_tensor->get_shape().size(), 1); - RET_CHECK_EQ(num_boxes_tensor->get_shape()[0], 1); - const float* num_boxes = num_boxes_tensor->data(); - num_boxes_ = num_boxes[0]; - RET_CHECK_EQ(detection_boxes_tensor->get_shape().size(), 3); - RET_CHECK_EQ(detection_boxes_tensor->get_shape()[0], 1); - const int max_detections = detection_boxes_tensor->get_shape()[1]; - RET_CHECK_EQ(detection_boxes_tensor->get_shape()[2], num_coords_); - RET_CHECK_EQ(detection_classes_tensor->get_shape().size(), 2); - RET_CHECK_EQ(detection_classes_tensor->get_shape()[0], 1); - RET_CHECK_EQ(detection_classes_tensor->get_shape()[1], max_detections); - RET_CHECK_EQ(detection_scores_tensor->get_shape().size(), 2); - RET_CHECK_EQ(detection_scores_tensor->get_shape()[0], 1); - RET_CHECK_EQ(detection_scores_tensor->get_shape()[1], max_detections); - - const float* detection_boxes = detection_boxes_tensor->data(); - const float* detection_scores = detection_scores_tensor->data(); - std::vector detection_classes(num_boxes_); - for (int i = 0; i < num_boxes_; ++i) { - detection_classes[i] = - static_cast(detection_classes_tensor->data()[i]); - } - MP_RETURN_IF_ERROR(ConvertToDetections(detection_boxes, detection_scores, - detection_classes.data(), - output_detections)); - } - return absl::OkStatus(); -} - -absl::Status OpenVINOTensorsToDetectionsCalculator::Close(CalculatorContext* cc) { - return absl::OkStatus(); -} - -absl::Status OpenVINOTensorsToDetectionsCalculator::LoadOptions( - CalculatorContext* cc) { - // Get calculator options specified in the graph. - options_ = - cc->Options<::mediapipe::OpenVINOTensorsToDetectionsCalculatorOptions>(); - - num_classes_ = options_.num_classes(); - num_boxes_ = options_.num_boxes(); - num_coords_ = options_.num_coords(); - - // Currently only support 2D when num_values_per_keypoint equals to 2. - CHECK_EQ(options_.num_values_per_keypoint(), 2); - - // Check if the output size is equal to the requested boxes and keypoints. - CHECK_EQ(options_.num_keypoints() * options_.num_values_per_keypoint() + - kNumCoordsPerBox, - num_coords_); - - for (int i = 0; i < options_.ignore_classes_size(); ++i) { - ignore_classes_.insert(options_.ignore_classes(i)); - } - - return absl::OkStatus(); -} - -absl::Status OpenVINOTensorsToDetectionsCalculator::DecodeBoxes( - const float* raw_boxes, const std::vector& anchors, - std::vector* boxes) { - for (int i = 0; i < num_boxes_; ++i) { - const int box_offset = i * num_coords_ + options_.box_coord_offset(); - - float y_center = raw_boxes[box_offset]; - float x_center = raw_boxes[box_offset + 1]; - float h = raw_boxes[box_offset + 2]; - float w = raw_boxes[box_offset + 3]; - if (options_.reverse_output_order()) { - x_center = raw_boxes[box_offset]; - y_center = raw_boxes[box_offset + 1]; - w = raw_boxes[box_offset + 2]; - h = raw_boxes[box_offset + 3]; - } - - x_center = - x_center / options_.x_scale() * anchors[i].w() + anchors[i].x_center(); - y_center = - y_center / options_.y_scale() * anchors[i].h() + anchors[i].y_center(); - - if (options_.apply_exponential_on_box_size()) { - h = std::exp(h / options_.h_scale()) * anchors[i].h(); - w = std::exp(w / options_.w_scale()) * anchors[i].w(); - } else { - h = h / options_.h_scale() * anchors[i].h(); - w = w / options_.w_scale() * anchors[i].w(); - } - - const float ymin = y_center - h / 2.f; - const float xmin = x_center - w / 2.f; - const float ymax = y_center + h / 2.f; - const float xmax = x_center + w / 2.f; - - (*boxes)[i * num_coords_ + 0] = ymin; - (*boxes)[i * num_coords_ + 1] = xmin; - (*boxes)[i * num_coords_ + 2] = ymax; - (*boxes)[i * num_coords_ + 3] = xmax; - - if (options_.num_keypoints()) { - for (int k = 0; k < options_.num_keypoints(); ++k) { - const int offset = i * num_coords_ + options_.keypoint_coord_offset() + - k * options_.num_values_per_keypoint(); - - float keypoint_y = raw_boxes[offset]; - float keypoint_x = raw_boxes[offset + 1]; - if (options_.reverse_output_order()) { - keypoint_x = raw_boxes[offset]; - keypoint_y = raw_boxes[offset + 1]; - } - - (*boxes)[offset] = keypoint_x / options_.x_scale() * anchors[i].w() + - anchors[i].x_center(); - (*boxes)[offset + 1] = - keypoint_y / options_.y_scale() * anchors[i].h() + - anchors[i].y_center(); - } - } - } - - return absl::OkStatus(); -} - -absl::Status OpenVINOTensorsToDetectionsCalculator::ConvertToDetections( - const float* detection_boxes, const float* detection_scores, - const int* detection_classes, std::vector* output_detections) { - for (int i = 0; i < num_boxes_; ++i) { - if (options_.has_min_score_thresh() && - detection_scores[i] < options_.min_score_thresh()) { - continue; - } - const int box_offset = i * num_coords_; - Detection detection = ConvertToDetection( - detection_boxes[box_offset + 0], detection_boxes[box_offset + 1], - detection_boxes[box_offset + 2], detection_boxes[box_offset + 3], - detection_scores[i], detection_classes[i], options_.flip_vertically()); - const auto& bbox = detection.location_data().relative_bounding_box(); - if (bbox.width() < 0 || bbox.height() < 0) { - // Decoded detection boxes could have negative values for width/height due - // to model prediction. Filter out those boxes since some downstream - // calculators may assume non-negative values. (b/171391719) - continue; - } - - // Add keypoints. - if (options_.num_keypoints() > 0) { - auto* location_data = detection.mutable_location_data(); - for (int kp_id = 0; kp_id < options_.num_keypoints() * - options_.num_values_per_keypoint(); - kp_id += options_.num_values_per_keypoint()) { - auto keypoint = location_data->add_relative_keypoints(); - const int keypoint_index = - box_offset + options_.keypoint_coord_offset() + kp_id; - keypoint->set_x(detection_boxes[keypoint_index + 0]); - keypoint->set_y(options_.flip_vertically() - ? 1.f - detection_boxes[keypoint_index + 1] - : detection_boxes[keypoint_index + 1]); - } - } - output_detections->emplace_back(detection); - } - return absl::OkStatus(); -} - -Detection OpenVINOTensorsToDetectionsCalculator::ConvertToDetection( - float box_ymin, float box_xmin, float box_ymax, float box_xmax, float score, - int class_id, bool flip_vertically) { - Detection detection; - detection.add_score(score); - detection.add_label_id(class_id); - - LocationData* location_data = detection.mutable_location_data(); - location_data->set_format(LocationData::RELATIVE_BOUNDING_BOX); - - LocationData::RelativeBoundingBox* relative_bbox = - location_data->mutable_relative_bounding_box(); - - relative_bbox->set_xmin(box_xmin); - relative_bbox->set_ymin(flip_vertically ? 1.f - box_ymax : box_ymin); - relative_bbox->set_width(box_xmax - box_xmin); - relative_bbox->set_height(box_ymax - box_ymin); - return detection; -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/openvino/openvino_tensors_to_detections_calculator.proto b/mediapipe/calculators/openvino/openvino_tensors_to_detections_calculator.proto deleted file mode 100644 index 41dce68853..0000000000 --- a/mediapipe/calculators/openvino/openvino_tensors_to_detections_calculator.proto +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright (c) 2023 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// - -// The option proto for the OpenVINOTensorsToDetectionsCalculator. - -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator.proto"; - -message OpenVINOTensorsToDetectionsCalculatorOptions { - extend .mediapipe.CalculatorOptions { - optional OpenVINOTensorsToDetectionsCalculatorOptions ext = 113776539; - } - - // The number of output classes predicted by the detection model. - optional int32 num_classes = 1; // required - // The number of output boxes predicted by the detection model. - optional int32 num_boxes = 2; // required - // The number of output values per boxes predicted by the detection model. The - // values contain bounding boxes, keypoints, etc. - optional int32 num_coords = 3; // required - - // The offset of keypoint coordinates in the location tensor. - optional int32 keypoint_coord_offset = 9; - // The number of predicted keypoints. - optional int32 num_keypoints = 10 [default = 0]; - // The dimension of each keypoint, e.g. number of values predicted for each - // keypoint. - optional int32 num_values_per_keypoint = 11 [default = 2]; - // The offset of box coordinates in the location tensor. - optional int32 box_coord_offset = 12 [default = 0]; - - // Parameters for decoding SSD detection model. - optional float x_scale = 4 [default = 0.0]; - optional float y_scale = 5 [default = 0.0]; - optional float w_scale = 6 [default = 0.0]; - optional float h_scale = 7 [default = 0.0]; - - optional bool apply_exponential_on_box_size = 13 [default = false]; - - // Whether to reverse the order of predicted x, y from output. - // If false, the order is [y_center, x_center, h, w], if true the order is - // [x_center, y_center, w, h]. - optional bool reverse_output_order = 14 [default = false]; - // The ids of classes that should be ignored during decoding the score for - // each predicted box. - repeated int32 ignore_classes = 8; - - optional bool sigmoid_score = 15 [default = false]; - optional float score_clipping_thresh = 16; - - // Whether the detection coordinates from the input tensors should be flipped - // vertically (along the y-direction). This is useful, for example, when the - // input tensors represent detections defined with a coordinate system where - // the origin is at the top-left corner, whereas the desired detection - // representation has a bottom-left origin (e.g., in OpenGL). - optional bool flip_vertically = 18 [default = false]; - - // Score threshold for perserving decoded detections. - optional float min_score_thresh = 19; -} diff --git a/mediapipe/calculators/openvino/testdata/add.bin b/mediapipe/calculators/openvino/testdata/add.bin deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/mediapipe/calculators/openvino/testdata/add.xml b/mediapipe/calculators/openvino/testdata/add.xml deleted file mode 100644 index 537e4d13f5..0000000000 --- a/mediapipe/calculators/openvino/testdata/add.xml +++ /dev/null @@ -1,83 +0,0 @@ - - - - - - - - 1 - 3 - 8 - 8 - - - - - - - - 1 - 3 - 8 - 8 - - - 1 - 3 - 8 - 8 - - - - - 1 - 6 - 8 - 8 - - - - - - - - 1 - 6 - 8 - 8 - - - 1 - 6 - 8 - 8 - - - - - 1 - 6 - 8 - 8 - - - - - - - 1 - 6 - 8 - 8 - - - - - - - - - - - - diff --git a/mediapipe/calculators/openvino/testdata/labelmap.txt b/mediapipe/calculators/openvino/testdata/labelmap.txt deleted file mode 100644 index 4291e3c6b4..0000000000 --- a/mediapipe/calculators/openvino/testdata/labelmap.txt +++ /dev/null @@ -1,3 +0,0 @@ -classA -classB -classC diff --git a/mediapipe/calculators/ovms/BUILD b/mediapipe/calculators/ovms/BUILD deleted file mode 100644 index 3516927040..0000000000 --- a/mediapipe/calculators/ovms/BUILD +++ /dev/null @@ -1,144 +0,0 @@ -# -# Copyright (c) 2023 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -licenses(["notice"]) - -package(default_visibility = ["//visibility:public"]) - -cc_library( - name = "ovms_calculator", - srcs = [ - ], - deps = [ - "//mediapipe/calculators/openvino:openvino_tensors_to_classification_calculator_cc_proto", - "//mediapipe/calculators/openvino:openvino_tensors_to_detections_calculator_cc_proto", - "//mediapipe/calculators/openvino:openvino_converter_calculator_cc_proto", - "//mediapipe/calculators/openvino:openvino_converter_calculator", - "//mediapipe/calculators/openvino:openvino_tensors_to_classification_calculator", - "//mediapipe/calculators/openvino:openvino_tensors_to_detections_calculator", - ":modelapiovmsadapter", - ":openvinoinferencecalculator", - ":openvinomodelserversessioncalculator", - ], - copts = ["-Iexternal/ovms/src","-Isrc"], - linkopts = ["-Lmediapipe/"], - alwayslink = 1, -) - -cc_library( - name = "modelapiovmsadapter", - srcs = [ - "modelapiovmsadapter.cc" - ], - hdrs = [ - "modelapiovmsadapter.hpp" - ], - deps = [ - "//mediapipe/framework/port:logging", # TODO remove logs but need better error handling/reporting in Model API - "@ovms//src:ovms_header", - "@model_api//:model_api", - "@linux_openvino//:openvino", - ], - copts = ["-Iexternal/ovms/src","-Isrc"], -) - -cc_library( - name = "openvinoinferencecalculator", - srcs = [ - "openvinoinferencecalculator.cc" - ], - hdrs = [ - "openvinoinferencecalculator.h" - ], - deps = [ - ":modelapiovmsadapter", - ":openvinoinferencecalculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/formats:tensor", # Tensor GetContract - "@linux_openvino//:openvino", - "@org_tensorflow//tensorflow/core:framework", - "@org_tensorflow//tensorflow/lite/c:c_api", - ], - copts = ["-Iexternal/ovms/src","-Isrc"], - alwayslink = 1, -) - -cc_library( - name = "openvinomodelserversessioncalculator", - srcs = [ - "openvinomodelserversessioncalculator.cc" - ], - hdrs = [ - "openvinomodelserversessioncalculator.h" - ], - deps = [ - ":modelapiovmsadapter", - ":openvinomodelserversessioncalculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "@linux_openvino//:openvino", - ], - copts = ["-Iexternal/ovms/src","-Isrc"], - alwayslink = 1, -) - -load("@mediapipe//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library", "mediapipe_proto_library") - -mediapipe_proto_library( - name = "openvinoinferencecalculator_proto", - srcs = ["openvinoinferencecalculator.proto"], - visibility = ["//visibility:public"], - deps = [ - "@mediapipe//mediapipe/framework:calculator_options_proto", - "@mediapipe//mediapipe/framework:calculator_proto", - ], -) -mediapipe_proto_library( - name = "openvinomodelserversessioncalculator_proto", - srcs = ["openvinomodelserversessioncalculator.proto"], - visibility = ["//visibility:public"], - deps = [ - "@mediapipe//mediapipe/framework:calculator_options_proto", - "@mediapipe//mediapipe/framework:calculator_proto", - ], -) - -cc_test( - name = "openvinoinferencecalculator_test", - srcs = ["openvinoinferencecalculator_test.cc", - ], - deps = [ - ":ovms_calculator", - "@ovms//src:ovms_header", - "@ovms//src:ovms_lib", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:parse_text_proto", - ], - copts = ["-Iexternal/ovms/src","-Isrc"], -) - -cc_test( - name = "openvinomodelserversessioncalculator_test", - srcs = ["openvinomodelserversessioncalculator_test.cc", - ], - deps = [ - ":ovms_calculator", - "@ovms//src:ovms_header", - "@ovms//src:ovms_lib", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:parse_text_proto", - ], - copts = ["-Iexternal/ovms/src","-Isrc"], -) diff --git a/mediapipe/calculators/ovms/calculators.md b/mediapipe/calculators/ovms/calculators.md deleted file mode 100644 index 5bb17c598f..0000000000 --- a/mediapipe/calculators/ovms/calculators.md +++ /dev/null @@ -1,32 +0,0 @@ -# Mediapipe calculators - -This page documents how to use calculators with OpenVINO backend to run Inference operations in the Mediapipe graph. The OpenVINO calculators follow the setup similar to existing TensorFlow calculators. There is an Inference calculator which employs ModelAPI interface. It can be accompanied by the Session calculator including an adapter to a specific backend type. - -![Graph schema visualization](diagram.png) - -### OVMSInferenceAdapter - -OVMSInferenceAdapter is an implementation of [OpenVINO Model API](https://github.com/openvinotoolkit/model_api) Adapter [interface](https://github.com/openvinotoolkit/model_api/blob/master/model_api/cpp/adapters/include/adapters/inference_adapter.h) that executes inference with OVMS [C-API](https://github.com/openvinotoolkit/model_server/blob/main/docs/model_server_c_api.md). - -### OpenVINOModelServerSessionCalculator - -[OpenVINOModelServerSessionCalculator](openvinomodelserversessioncalculator.cc) is creating OpenVINO Model Server Adapter to declare which servable should be used in inferences. It has mandatory field `servable_name` and optional `servable_version`. In case of missing `servable_version` calculator will use default version for targeted servable. Another optional field is `server_config` which is a file path to OpenVINO Model Server configuration file. This field is needed only in standalone MediaPipe applications when server was not initialized earlier via [C-API](https://github.com/openvinotoolkit/model_server/blob/main/docs/model_server_c_api.md). In this case, the calculator triggers server start through [C-API](https://github.com/openvinotoolkit/model_server/blob/main/docs/model_server_c_api.md) - -### OpenVINOInferenceCalculator - -[OpenVINOInferenceCalculator](openvinoinferencecalculator.cc) is using `OVMSInferenceAdapter` received as `input_side_packet` to execute inference with [OpenVINO Model Server C-API](https://github.com/openvinotoolkit/model_server/blob/main/docs/model_server_c_api.md). It can use `options` field `tag_to_input_tensor_names` and `tag_to_output_tensor_names` to map MediaPipe stream names and servable (Model/DAG) inputs and/or outputs. Options `input_order_list` and `output_order_list` can be used together with packet types using `std::vector` to transform input/output maps to desired order in vector of tensors. This guarantees correct order of inputs and outputs in the pipeline. Example of usage can be found [here](../../modules/pose_landmark/pose_landmark_by_roi_cpu.pbtxt). - -Accepted packet types and tags are listed below: - -|pbtxt line|input/output|tag|packet type|stream name| -|:---|:---|:---|:---|:---| -|input_stream: "a"|input|none|ov::Tensor|a| -|output_stream: "OVTENSOR:b"|output|OVTENSOR|ov::Tensor|b| -|output_stream: "OVTENSORS:b"|output|OVTENSORS|std::vector|b| -|output_stream: "TENSOR:b"|output|TENSOR|mediapipe::Tensor|b| -|input_stream: "TENSORS:b"|input|TENSORS|std::vector|b| - -In case of missing tag calculator assumes that the packet type is `ov::Tensor'. - -## How to adjust existing graphs to perform inference with OpenVINO Model Server -Please check following [link](https://github.com/openvinotoolkit/mediapipe/compare/master...openvinotoolkit:mediapipe:main) and look up differences in existing MediaPipe pbtxt files. diff --git a/mediapipe/calculators/ovms/config.json b/mediapipe/calculators/ovms/config.json deleted file mode 100644 index c9583066d9..0000000000 --- a/mediapipe/calculators/ovms/config.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "model_config_list" : [ - { "config": { - "name":"ssdlite_object_detection", - "base_path":"/mediapipe/mediapipe/models/ovms/ssdlite_object_detection" - } - } - ] -} \ No newline at end of file diff --git a/mediapipe/calculators/ovms/diagram.png b/mediapipe/calculators/ovms/diagram.png deleted file mode 100644 index 71941f7d7667f130c55f0276d772ea0a358d4b66..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 50805 zcmeFZ2UL^W(>JOJM+H$75QSsFh6>Vq?}Xk75Sr8wLP8A*p@|Bph*%CFU5fNxr7MW2 zNDsYgc+QdYt@GY{*Sp?(*LTK1kPM z4;(l^udb$SaNxk9hyw?x&_{j)ccPxC2!cNcJq@6W2lCr~pFVKlaEh0TsTUgWh;(r{ za9K#{=hbCFekWH?uggNpmjwlF+}wEWkv0eq8?+}c#=#5R0?*N|_DDyhgZTTmA|%Eu zBnoa{)l@Um*19aH1g>3>CR^X*rF2np@^D3iJE{T#qP+Z+f8eH}jiZeRa^Db? znK^jbAb!pY@2Cn_Q`0rnL!0S)+MA*@b&UZ{OpMQlatyWjv^{fjtC_f-rEIgYlgEo zcKkVr7|hMyPQ+7KRgYi55k-Ka5X#DCmjxBwQCJZHH!p&>vyqp#ry;+CB2t{+L|s(e z6E6gquj*)KY@o)ksqg8ER@A|wjrfgC4BbriuqZQOej{xsaYsj_GQYTxyPCcy0)tg_ zv(rH8i{da)B{j4O=&pitHo(L1NVtHii!)9Wris(m($w&FRo5_5wHHuTF!2#Ux@zK0 zRP=529FPPJ1yx5IWi?G*dv6Z`V|5d-E~#8XU8saq6u_A7|KAPSVnogQHK~+k7M_a6^vWbDGhP@45OG!<{K|#XKKv7HC zo#14n;jE4}QFV2KtGk%FD~cGHdJ3y+2%<4INJAej1H6rhFhWxVYNV?G!zv4_*uxEs zj1fv`EX>xxP8ABq#v1DSNa%UMOjLDz>@i3iH*pgKxP*&{vZkW2sXJ)EuVALDZ)2jO zjW9OmH-+NOT)j+;wVcreEm1?1gP4b&u8Es5(t%&k)6_{B0eUFvI_YX_@GI)77$H3c z-5vD6eo#acL>vS)UHBa}DJ`(hLMSJcqn-pt%|Txk>F&jk)`E+v8fgj&z=RD{Z57Q3 z9vV&pXk)aNg1rRjpyDL1VT9C#flDO^C8Q2oOjk*tAFpb!3GV0tIakGLL-86v7gklx zUKqu1h;Z}4DGS&s3h6r|j0ALyq1vW+jJAn3M#9xlUtL8<8;@~R@&sKJ^|2;8BH}o8 zdzc&C0q!oUrYhioRZ+F~fw~yNz+O;A+Myg_Fn1_MP~6>CO%Um!?rv{uWGtv`gYyy+ zP&7jt;SEh~yj_LBzE#xMM+zD!dAW&bTvWppvqUfuoJL zr?Z-%rizv!%*?>q7NdlM8M7IdoM4%h6cvX0FA*3+o-|$bv5zMHm+vM zSiGPa%G*T9MH{>ksbv6Wq~M{Xpen4Su4ce*Z=i4Ff>O}2b235tcq)sjI4ioD8RIeD z!lFvTN=Dim&TaxYj5h)gI*Tb7iD)~jn+h3d6ZBlQ?HzVWqsdK=+Q?F67m5vaE~6eT3Cfl(u%@ft#^?ly28ZGJU%jDfR&sfU{Y z0;%Ys4%C>UK2*pIZmh4S>qJ=^Aq^9xhKGl`F+auyX=@5}i>fKw!INNQ=4K@BhJu^m z-F1x2@WL7<&bG#G1SA}xu5PR2>B1{N7uw%*%YGz_kfx?1C^#|VvKXPLkNg^dI}+QGzBo~uI_M{7NQj$3)!^PH0=nS22Ognh=i-D_Mw{9RiJ7?I-3@iH&Tug_ z7#wIkXQ-p0h>#j+=xpoZ;Ozi40d_%E(Mj3e1m)ra)zfn`L87rxK}Ba#n3~sjbRv!r>O(ZMFWdd76)B4#1O8kU@2Ac7+omN8R}_bZ{i_j09WvIagnfv+j;>T zsA#WZgLOpd0uuom{$(ZhOa}P=r#Ti<+2YF6K5&5JfV#4R5#H+i$kBKmhu!8MV^3Wo zrIFE3L^R6pKl$VG&?trroEO(Dczi$unVfUbcBXaYnLln@?!7*_1*4z(MMLA9tajtZ5^5 zx0C|D7~VQSb%f*Q-yaNSpR9K`qEABBz4qOuJb%M2kN)_zQg;uk(WO*~AbV|od%z$BEarWN5%Lb^>A zoQ_j?D6f35;UfF7y}1}MihE5x`r4i}w(@;|5fzsph4W_o9Pj@2?8zZP_s_~GKwa+G zNWkW7xl?C~sa*Hi?o!h3o2^`zf#N7`6JD{~Gw~9hT8$2_)w80$Q^}#WwXT~hv*k98 zzYUe61}Ykr7FoZjA?ueZEOcOronGnwd{Z=OWBCWL7F9W^pvx0&z% zqlu-r8aMwe3HQCfQ}yBR(>wX(mHDx;))m&Vl;_#^2n~mZMEWckXDYSHPj)uPcFRT4 zgUS{ohfYY}uzUNU2u%F#hJV_^OZ*R8)L?1W>}c&~8g{x;x3vRaOCTTHxo5iyMC2f~ zf#J=CXR(7W6e)Q3iRy&hR$(~1oNQFXRA;I#?H`Yy1#FW|r0@$ja;>wX`PO!t)yDN3 z?GPWIot%+Tz)KUBGeZp(pS(^d6f=LVt=ZsODbSCOd_MVDJtvH zrGxYWGH##bb62wu^J_+3SBG0q#ibmLV4cfaSS1?jsvIjOz0WZ%O4(kYOq%ogu;7-J zs7230AGMI!{ArSqzt|=<@71nT_v0;FdBFC@(-mF0>Xkv%Nc#lICs#i;1Uuc;L?7%w znSisUucwE-bZ9wUP0G&E8Hiy1sU9Ca0t)c_`GAN$a;{U=Z?D8z;>UxFdS4{R)o(4f zsLza5<6MnuA8%Ae+&C5cvLUrR`2cR4+}AA4o~7<0=uPwMW%Ju`W{Z1wO9cKe-JC(veUD>V7oMNhraOK849a}m=K9(Ts;`%$EuP${hUHrz9X|Zw zUJ%AD?HtlWMME23${YNuBgRbRe7nzN0)ljOq!js-$F%SjVdeW-SKmfoNKeW7M1Y*u1(3t$dk*OxdtyZj>3OX{j@zg z#z2#1->PBXQ$buoV3Vl)o|tX*Y5}qGgj<@+@7kTFim4n4$F}%s&bCfv(Z`Nr^rK07 z?WrL-yp~nMpTF_et$j8vX&S{(gs)6zB)vTgso&X{dD>7;l^4<&!h5GVQ#PRH$lma0 zf$Y7%`HpK(_EH#+u>Y2gyLL6CO*%UX>DaT7q%x?eC=)@b(|1bkcXaVvE8EN@t+Nba*{xBLd_9$u*!ex%wQk+<;|!n=OClCkj1p$j zX0xIVh0;UXgnd^>a6{h*N=8seBYx-Kd#CIX28e&=DYP=!?p^n8)+Jtec22}ATKMG0 z)CgJDNh_Ua{IB6<&e*PbMv~71AZ2e2$hSgxA3O4Be|O1Yy`OzAsNu@TdqlhbN>7CE zw;05ci0LmStr*JAI;#M-`N`~$vKPP?q?&v(^2@u&SIX3s49I?;X;>^6mZ}>=PYe~> zG-@^KUgS}ykD!elpE;HkY0Nl9EmZm25U<6{LgcN48~2uFB7k}?iMAqK`zAsNDA9=I zy5iqA(ZW~nq8%rCSvozrol|>HW8Jj|7V--9p8OcotjrD}Xl;B~zTbOkNpEL-b#pwF zefU^rvXV^W*dyYC_+RF!2U#4aaQn9b_P{NhrEjgCji|<2y*?MEH4%j}(9Z!1xcNWjx=WaeP9(Og>AHT`-DH3xU?^ zT1P(azw8xaR4ZoCZYMlr)SgDTdOH{aN!6ba*;|7QbFitZxL>VMyn$m}&nMa0gq>~x z;}@p!9LN&}2CKbd`h*oi-r1b1ZWq3d@@t_N8~#L)Q)2OFNh3fn21A2eUJ{IMW3LL= zEi}+YNq9Oo*cnAms_YGqq-a|hmx9!u=W+Epf)y}iI`QNK6h1rT{tBt5#b3MmTWg%Q zP~vYbzyxA8Xpix1I7f8Irzat&vU=H(oeKkC^Y#P3KzE~zaJl*6Ih60}9JZZ1gtyGO zN01z^N*fVN%m5;vZz5oD_n3d1fLFp}PHIW~!t(_Gm4Tx2yaJKR?Ufyth-i&s0?G&=R{-q>-h< z_%)NoYqBi?+T=oro14~fKSi<(X%Jje-A#b z50(g$^Fj8Xx8Gj}-jkyqY)_T_P00UeiB7#@OmXODyLIKt)Sq%zp2kgeyaqZJ6(P5S ze^c*UY%a{Ww~~K;NS_2UVUP(ty8b_|8M^H5Z09%8&@vSQgE{aqM(EH%7S}FG7uW=7j>tq0Z=S3<(@4Lq!|BCToL+X{_`^1kKiG&XuE3-2{ARS)& zn=AlTG^P3CLaAZXCeSp}qlaQ2?w?*A#TsQXjdxCHbnfrC#R6Dup+tbl@ItgHXRkj= zc_jqih@hcE^MWp)|ItO75p;p^H>5Vxq)zW+79I%gat&VV*@XK+q~typK#qc$a>4M1 zXUj?MKb-bIN~ApEHg)nm8UbO6NZ-eyPcdLcOKzP6gq`^ZVRnG9$)<@IXat>F_r6tG zPy-y$FKUSUP-q+x2K~n)E%3gD_J- zn9mRVx6TQwuKmj*4U{RtD0d-U(52^J*6%qkr)D zG2n5kZLnCRP0qiwKUg#xu;$;^>&X(eFIzpfeR$@1VIG#3Gymp=da2*fY#AL4fj@Ho?ST-%{P``_80+_1FQB)`4jL!xUuB#J?@5(O$=bJ}IX;B}UPo(e7bF0rf#MUu)3sv0XCws_ zYx30+AD)wP1eUcT6&gV(7ZYaLx2vfAfCk?%$l5LQeD~-)ILNz71#G>f05!*d7r%i> z^hqbr^tD&v=H~`RJ1qXvITY>lryf%rjFIv6mh$+b_EERkv6XdZ9x``Id%qeF66OUq za)RbQQJO5fGTq%wZ_*ksQ8I)5-kTQ}>Z|y4pK6eQ#R681FOn98=-5;%NX?*08`w8b z{LkH~;CUOt4kn}v8i@R(!5T%hvf$^@J&$@$^T7M>9A&%?j?;;AG_CtdI%rMZWsLvM zzo=wAj6|HGQN>m9+?)S`-uEgFH!iAaUTIm6U7n23uYJ11A~vPXX1KN2rXvGgT~mV1;po_Y|y&+I&UKym8c z*c)F1fUcIAb?-ha`X_EM3QTG3HD-F1zpbTODCp#6N?fkeywki4H(S|*6!#v#It(Q5 zgmFr^4)9kG@42-7a!UKK!2|Hr(Hn$^rvn&u5tIx@>W=FLz(Di zZ6qxB4HgN%z3?o(MQLK6#8(1&V5XDK4a7<9HiUhjWXBCP_>_F#qT?+-9_7EatD>h>AKDJ`oC3>=TK47;oKS8t*cV*LS-GCcbQ#zublPE&S z3`dV_8RsTKgWm_$N)Din=Ssh!*-Ml{j}N|kU5g&*TD^|yb%E7-`=bWx!e-akp{Tyx zIZgCEAceYx&m!WiCZe&h4QdH*6&ki+=cqq#ZmBOl5=nxNjAP1&$A zRZDZ>z;Yh*s;lXJ^Y1aUv$HXVxyGAUzTXPtEn-Pzv%OrvjcR}3-@0sJ*Ao{8U-^zO zE9p&e*&gg!`9_RJuPqh8$5WbY8z!tnPng|i6&oTuxRwj>wU%BtD}kq0a}Ru@w;oi= zgOuf(l?>i}T|2uI3OApeJLIx>M_-hHm-Bt8WKvx{emkoElX2G7Coi>6LKr8Ta`uUk z8gO;(Je*ZeLh1*~2cmK}2HPZOTsCLGdCw^Q`Hau+;o~lKC87DZCdX` zzPNI<9DBt;_Z9OBXxxw#d;HX8VhVq(Sm1fz=Vsz%a^yCAGhou&*LBooRWi6adAn+W zt)R{cRX2*_f=`{2UOKboLcB8#%Y(_R_C>#ywbNalaXwK#bpx0(oZL{^lDRoQF1^*V zs<-W<7bg%txkE0rzqHXoYaqT^rV_!;Tfz7H4Rz(%OM1#r^psf#?7Vm6*&@P4vytpP zW=<7jKGBw*=xdv^eKIQ>E$!6{{i%kUVqdWDKcKIai64n@Xc=*aMnKAv0teA_QRu>w zcG+fnegE;w*O+Id`s?0PF1_U?+x}mU{GX7pyR}-MAFUroV0FFfkvw8n9pw8^zY)55 z4B~J?7(EkE(2=o_v9Pvv_u$HK$CcRHRcyh<>c1m)4^@z;*50$&P)yBqZO=gUYF+vP~2)#=GwMQ8i z07Uuhek2{|$7#7On(Fmlt`*XqPEq8R=SoA4tJ%FShwtgOm3iolL|(Ogv)Ts9GK_ub zafq9e@#@y%Q)BMH@Dcrz{IbB@iag8hEb??$Z^*;MFNyBQwiBBVg58u>4FhSPaA1e{4GPH^cxe41=^Q!fxUhBtCxl(W))w<47VDpe{5U@|viS%*C-91ViAy?QZM`B&s+}3bR$-YmMiy>+RzECjD%?&s zaE^m7GGBGEvyO3~p=Bux+*xm)EAWi-FCsCri8>bdWQ|>ARykF1t-<46ByYR;!~@g$?WKs?sEOhDdvFAi zO*ft86y)jACp~}T4In5$4ffZmjwc&Y9MUW21SPX_-gV4hly;n2ELr+xOELgn^Xxv2 z6Hta<`P?F9dOdG&VMMlF?k_wGaDJcoYrHjJ#g+>k&z3jcX+CC7@cw1cIp~3%?3O?2 zqzDXf^He;Zc^?n1QM`wv#=S8Yz%wQrhZnzIeMI@PJpRG`dn_+#0HyFuD)cW0;$0?a z{s8WCVl{4I%BIt4FYLHyw&bgT$C(~ykiL8Fymb05Yf0#@5s@ZfGTtbqmImPb_+8p3 z(c=DKylzB$6eDF#E}h}qcLL@(3pBv{yYI6NV4!<(UGBbPW6&C9B-t7YPGQK!djD_B z`RF*s>J?^AW23>}vf}T*r1Ito;D@khE3*L{>+uPyea!ePi0W|xbiPMD@%0z!M=qrT z!+2J}%=iOcWPXq78bSrOTR7v z*A%2d~Be;P_X9&Yvdql^CU|dAUnbib~kvhe+-`;Fnx;UT9*m?gw{r zK`1MuCc&)4aR4MTd>{3lm-&iKc*qPOfKbVyeYiJ=9qlDeytK!cyXfJUoiy+MnDdF} zx48u(U>}~?r91;kKm$RC4u4O;js7U!xC7;DQBD12>=+gdILsO*@-3@#9a^GRK-$fK zv5BQa8LF>5gr02>WSPwGw8V)O(4XOdg#n2$=Jf%!l*9#isgun_b3_G*1DfT*%UsqL z##iP8cZVF?6K4bJKqUK22MD{FJ-(m!Hk25h80z^3Xc`RTt8)T*OT`r6zT9F$i`^99ZYqq~Es zf=Wm~2(!Nhn4fdd?S`TTR|c47G%uQ-RhXZg;@A?^T8jYTpeT??S&=`JU)`3jli(c2 z!sjg)fnq-zVFmFNX5Dv)AhNJ(6LycW84!B1@R1SEtI{Hc5N98dtz$T{ldlUhBH>>Gd2A)tl@7dLBf+ur>zc z`marDAHeN7!q@}WUVBWo%@UMBFd7wlg5A6J;fDXhaArdVLDswiGs}75ekp*Nq`x@{ z@ADL`Xwyvb-#b%7XY`51AcfA*H7+D`2P`75G^w36HcLfGwxD(ezWM3)35$6v3z>*} z^bzn1!V!@g<3@{~LzxR1$z|{h!BCo7Qcq6x*ceD;;l0H5iKYe0?$6Y1n3rMc-ucajK_<0N zoRvT(fn@>0f^NLC#qOc%K2P=23;FxOPdi9DFdY{lDvY~>JOf_0kkDp}N{<*VPTj=s1Z!r&vdrkEnx(d| zwebtm1fP$cPQQ#Xbq$j^dR)I2`}&O_tt}y4Pf1>+=s}3{4wxS$;xCrA^u46r`#ew4 zxc-<-)+8gj3}RMn@7{jGIWF4LJ0{LX1mqt$;fV||dZF#5b~(!%#FQIgkBn80s?lHR zc_RrD+q~)x`CiEq z)n&pb<2@g;wPo2Pv-W1ccVQY8rpqBr+*-&~M?(A-6vS?4-7*IN{pUM<3})o1=>ZiP z!z_+$1Oquh`*U}Vk0;BTut00z&Jz3TM?kHS9RFSdfDPoTUn=K> zetSBBUZgua(f3$LopZl)>A3!UHApcSVK8s)sB-S%V)0^5C;M@NRh|gjzsip=)K5gD zxfZT6g@kejD{3307EE{WYXvK9uY&9lNi}WWrNtJ$>iK(=tU2Frd}nO4IbI;BT&t!z z*#vCwY{nkX(`5XRnE~=83m#OQiRFN{pZl%l9Bz2L{KZieKr8KluEsVZqIt}})zMxO zmL|!>@;{00qEf%Fi2+(`2Ydw^_ zaJnxAqD6j0ih0YO|MEg_3ngf>k$pV^S>~EkTQ{WB))ecbF|*jY`N@QY>{xjiclRaQ*Ycz^l|!_CSI%iYqI;kvrig# zkkN^VSd~(s%;(nMYVuZIvb0HGX#+<{^Z40cxG={fcNzm3X-5mvmod)Rc>7l4Z+X#UF>=pBjw#6bUl8w351!~fseGXIll16=)|o5J!yQGh52?V`HU?$M0@ zCkVZVrv48Gp-(YG#Q2=1S^_$CsCB~RJ4xNTE#3o8ALt(VZiW9bcEnts0}>?KINx|U zD3ki7RT#ip&)<}PCNJDqbnkeCN(=5F@S!gpaj}vmrbtY^S~8R$FH$YJA0V0t^X22; z{rUo=OVmBcDnCRUVNvjcRg6uM!}5r-?>S(HSiU-5>AEFfde{=~#En*~0XXCj$C``l z0Ev5#h&kD-jQ@DV*pdCDnpKcff!)!F_cLvSf`yd!zi}=S`hW)?-M;yj5mL)%lRXDI z8&?jTp_*3Y7@RQ$LCULQQjbqlr^rloDVigdK2y+Av59l<0R*ij+w!20v%yX^Q*iUH z{37j7YOHX*Q9(4Iy&(VNzG@x(a+TkxSmMHz|4~EiVG!Q=(E64^jfJYys`(V8TcvW~ zUMUB|tU~nZK@GN3d|=R;v-4^oR&2Lu=6FZsBt%FtasW#G#N;P6i&Z7%$F5V_2i>a~ z#Wtk&eO;>iaR)lD`7B!sbT-C_hR9bj++C53T6o6$p;af1CdlbTRcQxo@DUyi`neSk zT?8oP({{m&xgcV@m(KgB<|j2z`x43=ZUimdo&)FCiruYAIZ6eNtI^8b2uf~i;kKpk zY}v`7t=WpPDt|7cOug8VjIn^t)@5sOTzyg&4no*p?H%qhy?ZzQ1Jp5;V`pI8riITF zu)Sti-cRGlDm^nSaUPsQD(BRRcj7TGF9j#TE1wCJqctTn6*81pm=n2C55Ks{YK&mq1z;)bCW4p@(`C;<`x10ow#_{ggZY z!0C<>a~)L{5xRywFU@)X8OudbmIUgx&Tu^X049+@n5GnqR8@byRvYg-SJ?}Wv@`uh z6W7qBKI=eia*o$@*I7a7KXbF01*91vy3?smak}~(A`KHDm3J1T&z6JfNShIUcpaL3 z6?A9?6iLK)jk1FjSy9aJ6qS8iV)HTt^3;H(Vd9$q7rYSFSpyJ2IN#EylcYmgQIMNo zPRY(T)Rl1nN1C!eE8kXDQ5UC;bzQk_dkSvx_#zY@b=@o%)tB!LcvbyL)bpEG^;AmO z_xD$lNy4DvM=y)+X}l;a>SfpD(vh$hsZ0m^@9sA3*LgDI!36 z7_<6%01i|umoGXK&t?@(si}C$ey5rG9{wQJkvl+ntgBJv`Ukx^eaP)^AiXMB(bDv) zFeuw=1I1;tw;odw!STrkCTFHMuLM-CRDJ5D947;qb6JI*KJijB zUuS;n<(?fTw5_!Xwq!YV(mkS{Xs{zsRUsEF_Zi(opcXQQLa082imeIW6|bNdbUDos z-;Zx2*yjf`=mab+xUTo+nfEaj>{;XKg zD}f#qu8_5qpJ}I!qkvFZRFc5Utxc!yib{hG*iU54h`UO9F*8oiVEzE3wm0462|Jb7B$6ADFJ~s5W6RQNi6wtvS(47$)fQ)V(SY~6j9ks<8{2t zGragob(={JEYCy1vY^lN%j?1(uy-q0r}Ph*P5ZH!#FgzVR5|T30)XJ?(I39l3`aoD z?qZBQM#X2N_U#(EX@^5zv&BFjI)`O`a!&P3z=3Q-W)SG zj(E|VMJH(KcI^_W0m^a>;8*vLMv&fjkM%@>%9Uz5A)(CGT4l6Wr&Fp2gxC+*5#G^X}qNmN2w(zmA$4J(x--1Zhc?U|q?kvcC?l7$c z9KfBkgh{XX>9^{jzEEauyzwHlG&4l0F&)F?zcHPWFUAk%Vkafg2ooVFHscp8QrmbW zBc{H^s&Y|}>rbkm+fAM{2v5}8#C|_4)>|rjZU`>ipfo8n65l0py^g|sOvm)^kUKWy z5Uwl5_=-~59_3`ym8!}YHw)x)!X+y>iH>AhPH%C6aRHK27|Vs5uXZW% zUM2=LhajhhRNh;{!`w%dNdDa;{+2i5fpF&^<}ek0W87%oWB4q7+`Tb`evbF$tyJEd zOhjT>R@k$SMCiy!Z(wpew{fQ4>k)L{BNMVCb#0(RoVCwaW|q5*tSP4fD{MP9`RlA4kyGlU#SpVRY>Dxte{Dt(wz z+ghc-+^5y0aQ>7e%gpcjkI&N*S*Z%-P>&xI;9pad^CS_Wysuk}J|UcS7tSauqI&~Q zK~iOn@aI8+1tqafS$s4^-hd*!$t*S`=57@Q1_-(56zf~o>s53GQf;goOb`fmJQ;HH z`C%TQOyTu}i%bWwljbZifc7n^H^O?tGLs9Fm1>EcG^5eh0RzFh+PRFfjXAs?iid12 zwST$=LmzRpC90SK9XssCKQhwiX2w(Xh^QwbW0!Vqjv3M;x*kjb$RpV_6DqRo!%coQ z6_fTUg$Lh$77;%lLc<7ox-mWJe~pJh8*fxl8&WtwP|_UC%R@M>uOvqPUi>f^RCwAM zH_C!aNFzN%HQLkXuFyKtI+Gme&Xs2`MPi7V(Lw+&oH23z_T~N4kw-hA^6w+s;Jp51 zv3TOlnNd!9ZR5D*!0;5kSM(>`sstiCOs-s$ev9MnrN1h}mcQM-u8TH|^TPBk z1ek@C^SIG9q$Z9NkI)9&kDQSLSl}s_5-VN2*7U^(WPr7TRDc6iS=ZgN$MqR?Go$r2 z8Pb>-de}1EG7BP()F~=9RxW?HHvmISb}xv?6lAK@`jUc(m!yG{5q`Ej+a`;&-L3NC zblE%wB|U2A{FHEb*}}%jjZ*e)>FVgG$pE>HI7x=1%~|YdyIHN$&2-7}QMXc0Yim&y z_NQqo38y$-R&ohMhR_c@r0*eFA5>qPur$*qIzJ5wyT*^&rhNeSO3pqizAU+4P6?gN zI>bX+R#H5%M&CAO3UD&6v^OLZiJSRss&K`KrI#Ujq_efN)-SC8$_2O)WjG?=CCR)j zEU7qoMB6J>rCoYPTw4QAl=QNLwhN<gUoDA={naifmx{+wM zcQ!5E`U0lUVq|pl*&IkGbuwEw+HN%15kqPDl=l@hsb#;)3W1Ik^?u=DRm`4r5>~$z z@_f}obCZe2&@14L=3nssc6&UVgzmT_ zMO!@RYDqog!&#=#Za}(aqO(!$7*1Sa@^g5x5~BMol_CGV7|}O5*_BWbFMR$kfs?tH znYBkVIWgw>q^e9wtlp(Fwz>W`4e@D^vo(AY)L?&-lDph6n z)yajxw8*&AX0v((G%=gJXPADn+T>^8R#oYr7|id@g6Jg#F8ig2WRnCAk>Sp7 zLfW)J0lS-bLET0+-NWRurOczD%NFG!XRO^`)8sdc=8J~*T~0ob>;EOD`3_g>{7HEZ z3y@BWpb9cNE#9+`c{BCs4X^VlbmuMOr>K-Q-E3aSp;B1rtBnfaZlO|VvR{OWkd59} zWJYVVh!GsGm?rTuFsCRc`53+GR*6weK23kon1L>_n`jx(`Y!X;+3Tcry7buyPHC=V zB&zGC^@%1#A#+-PTAwO6V#=c&$cfpETS>fdk%J*29mvjOp;gZf+illOuuj_~%g%YS zELr2Xd_n=Q5pz0LAw5i(d?fR?>|lca?cjxH9}Y9cgls1qw#uYXu=t3c0p932v6ADAK0}R8Dw8m^ z)#N3fs%a*zUdDKKPzuXa@$k;+;O4cqCP^GMquaDaZv9MrT1V!}wX34Yt~Y4a`o0b; zvd#z7aeYVH#@CmuN#}Z~m4B2;)bJ8|g_FUA9x>EI4+Ob^pFc=CN^M}qF?gHTpv3ES zO7$+uorvQj-Qr!7ZTv*0^(6TdgDv>K2zpu4N>^1wEaX)N9|o{5YmsdAST0wEvG2I( zP5ELwGpb2d;`f(jkOXfiZn(!T8F9MI$uugYs;TU1=5&hal!M;$o?uJ7RB8R7)A&w-lK~b}zo*Ej*a7}`Lxz@S9jV+eQea3)o!}nYxEBZNhYWM}VMP)t= z)k*ML;-dJD5~NO|Hg`ga@Wrwn!xJf|Wf0j*>1{p9sVt{9w3MAar+jZ!DVOLp*eMQf zK@L(k(4^Ep?D5exytU0J`50;uAOB$O`FVy43f__?0(_=g$Bg2s6n;O!XC36M!yL(p z?+ph67OZ5X&U{ong9J6~dP+9HSUw$ZRHWpOPgjKB1DMT==ZPB&M?IfZVZvSR3P08= zl=UyL!}DvC%0Cm5LYijGe!M(Q&(?FSjW!~Kz^gM?e#P70qTn*@LEpuj@}+%6ARV9k z$fc)yRJ0+fE(Jq3v1o&^#jo}6@#nQf#p=|i~S-hbp0=)n07Ilj|QOo6o*_tprP304eZJK_F3crr_cVdByT%-`VKK zq(DfaE-b%<759fOGMmY47bg{1ie;-SPYPUbU9<|w8u7C`+Vc3~&b7|lAzMG11BS?r ze6SkRWJ70jkGv~c;dfV_Ymzp6#-^Fou1b@J7q;gT%H5ZKlfq9uXZ~o?LCeBrE7A6T zoHgLfJL_`0$>Y^i4Pmoe7Db3RJ$8r8%hBE?ZQts?2(BV{f@zs#r+v!&H+M?e19v*x zWH*1WzR_U(FVtqm6=TU4tTZiRhspwsb_vxS?3O5eN-$Z8qzV-8OF zb_O2vUbTh0uafWXew$&nZ7&GyHx6DL47wdUf*lc2S*k`rD*~&Bj-T7%{x+|`CNj|V zwsmZf*@w3%aCh^~I;>@9`!IU@t*grtECJ^^AfmEp9<=3SD`URfx?<_QR&eo3t;ziE z=Dc0G!w_zc+-GITuYPH!ZqPB>!mrNKPk*cAIqIwP`J$T`>GTXh8tTn6M@nXJQ+wf5IR%T-FVsf=z?eaj8 zVV+^d$(8q?qjN0=POTJp7F1k;_;-Y?1itx=K5&4r!iJ)s%eNKcJaLa)ck1h0$)Z?f z>15J0DWDEFFYEi|GW<4NI$}X)Z*V{IvdJV+Cqv0c} zx-SdMc0(gh5qx4dwmVY;s|Qc`&h?mbZXDB!=^x8Bx*m7K*E(q@Uj#GIyfd)tbE_bH zp5~reI}1u|DCLA~$y|hp%nNi#odnS81ETNl8yIucEv!w}$>rMht(1EW9M0Efi4P3g zUfFa7duOu0W9MS-+jC_W&uh}P&gU4W=7+PgZVvPoh~}5Xt{RnK5(tAz;jw0qXXsv% zYeaYGbKI8DMJpw{-Tvs$ITLxGc_Ll9=nRKxROb(l%#x?CJbZm)s4tq!u3c8veBN$H($DEr7XIW6r~vOz;m@BsniX z-k^`U;eWW+DGpYTR4Mqtf-)>)^DeKXlPy=0S3Tz&flO?XqZM_00qD78HPPqR< zr!$*WN?aLIU+$ZK)l~hb4~%_oud)4co}(4Mv0k19M{7CrjB9VzN|#4iy6xP%>1p^i zePw`DkZThCt+(DJ&J^wAy94#^^N*-Zx=h~qvX~;0>pK~mi*C7-G~0)glJ<+hPWQhl z)fHj$@<7vJ-mr_iy<;8o&8Ti`jp9LRL|x6s&Wt5`N64W0YJ|C?N)+d27&nF@eRQ?F zqEBh2o~{Kj@NLogR>V^Wa)EZ7>xptnfolK67T)p^xvkGTTOXh6^j7+(W|p6X*JqA0 zjH^i1ZN6S)DX>f8^=ZP?j1N+q2>B6Fycx<%;4+=RU?v7?2PFa<6Um) z?9hCYB{OKhrD7cqW&)AlQ| z%`O_Lp&noJzDOQmdX&g6chMI3p#l9>ufieSry!7wR=@T+RBtz2tYES{Igy{{+{jvA zdrw#erpc%;3mg||LgvLUxRYr2TNAkq`f6gg2uwD0V z#%zK-zM@++7uwGxJVV!s%qvxI?lk9_gFS$YK8;yU!4w~3->#w$_{JQ!@__?oTHt&u z2e=tOhFdUY0W$3657G0wh5ns29LY2NH_m1J4k0UCGa_Mpa~ltd`rT-LI+#x9WtllY zx|qLRhfBgv3w?k~)Kzj zFf{Z*yi1QO-(<*P7pv{`1kX0JIc)N{z)S8QZObRqrA(4F-4uJ<%kyKfAQQ=N7FICz z3BfsF4Sfg?_IP1&vxncD9K!3jYY2Y6BW%PI$BawqxTm9Zn@yHDvc26Dji^4foCP^C zlyW&~;M2*1{&XIx?@=C?M-J+J9&E^use0O1I5)ep6Phy|=)+h4=C@@Gs$#_BbN#BY9 za-?9SsOz|gnBPm=;F&zvil{kNB|QE?j?9LOFsiTJicnX<%;jeLa#XXR4&4IFEmP-{ z_UR&==C!intKWI(U+Nb=R8WmF_S(JLYX0)$Y(WV7m_hy@t6c^*tS1= z4HYdxR|&I*Zemaz|9mc?ByYnu6}?TER2RESjCy&zDp@JCYTmfJ&(J9wds>d>Gi^l4 zSPU|u3Rf8YR;!$g*>3j`Bemx2kFNC=^m+!ndFpBU+2F^Fd4wL;I$|HXrzJ z>`Vd&a+ke9Xt%bqJYYg{X0$Te*VPH{7dU+(P6n6WlMQQrTPyr_WvjdyQ()Rd!1;fG z6_`2rA6_XfsBkSY|1jdLOFnahW|6rxLTl3l)PNgN!+-v9lQ91 z?kjE6iP1hH5ZVvhymjT&>5+xFkP3X;~4} zSZ)W5Z_W>vyO#I2ioptfXGUDVei;2;=aHha$qCEZuv~fFJU@iUF?soDG{anu`&s>? zvhN+l)Bc4WytV%ad*>O|K&fY1~vDni6Y z4Uf5iE>*kW-R=`h}Ncq|8f;XwDC8Nq4nKj5>$*%)lF!r zD`Z)JY^wWsW7>h2;aMXOr-tIX9;c2KyDYqfSr-o#U~)fbD9q;SWT=^sD!;~Tkb5XY zIt!hjAhqxL)Jm@Rb9x3Vdd-N)PIsFOOFRkU48?K?_8XyUE$3ScD~xJus`7QKxAI$d zBrI=d9$1`|nyU5|M`tE5^HJey;CZ%z2ipC-Y+hcqvq?y)u&2H<-1_J^-0ER&*UsJV zPI$gJv^X2{7ODbv9k!_Huj-|5`cGE8X=cQdEf(Y-B`xe)ATZYpTxuEb{< zkAzK}`O-|4T}TY)ncFl_4e-8yS@y(bZ7lJT){au0uv-5**$9KaR~zS+vDtyF?v->J zqR7O&v-OhaF{^8gKp4_4VF${LB27hnFf#A+bO~Efmd#`;#GT4m(mw4dX*3Nvx3D0E zbjRx0;nLs+5el@NY8QErN!Q!}0`t0W#F4APwQ>aprkDxz^6@ppy`>4rTyJ1#D~mKv z9!q^lqjV25`$Ruff~G_WXP<;Tla)&eUQ2+2g}KORVDRjNYK(@g$@cmxrJ%{#2YGqz zW9`B)al<80#Uz;OgtOxMtn94N+FgTONhdcm1KE#!DEn9W1eNuj^-J zX2#Q_%kZ@gbgSr8)O1oZF{jtX@x2SCp053r=ZKGzY(;+Z!tD#V3TJxQ_40cF0`kx3F_`*O{d&_mB?D71q1{t@$?3qD32@PHbYeI8FNB z+0QiWra4o}pGEAJVBxP`oqBv4K(T72L~9a<9azTa7AcSirSDV)vuEMiv=p?<`|FRk}ZT^RZ*#3;D>D1MXp12 zDLP&?7T=}h+{<{*xpkTVaulxWSLg@``tPV0Na?h0GSB7dH#$fLjF$F|7KDp}me+hb z@xHjw=(CCDnjww1x0UX@r{|;_=>n*ctyftz>8XgyAHX-~yEh)mpJ$dDd~Yx*n=O<& z=04OW&{w`V;bQ2Fg&_zp-?tLshu8gvTxyGJq7rX>&AGRrYVFyw?dR`EA|J2LY$0Y$(r!}AYgs?;g2HlJ1B zNfOTtR0JjlmOTnfUM>{Nd(^@%Rk$|PKo>dCTBz>E?=#r&Uf7Ln6LMe??53)x^#vbR zcX|AfAtgvUP#S2xQsyYpM)M+L{MX=I)!yKks&fb7S#yyA;+R%7zno|*X zE*nq22a;v`MJ2f99N?#CqOV99VTRs2^~v&82D3kN#5|grRf=+`dUV6L>AVkbu{+*t zYl=YBU)yAFd_aBvy5M*5+xOGnW@yTYrd1X!K!Utp=kIwLx2=YvyYe^A!(5W)cH3X| zu4dUD;HxPN$D=cK+tO~OjsY;`%Rvr<{<0V)8I}2{uDd&fvF}g{L}T=5>QYvfV4v&u zP|mRa#fwXAaQ_#0*J1qOb^E!3`V$bUQ;PD(S>p%&%zTB069s~1iJdrzoo^=3(@U1j zgkG(NVs=}!qe#^E9_EdoZC<)UAlrDz@b~)!blok9egW3q;mJ2F2F?BpnU360oP8H{ zxN15H?3d(Pqqorw#L9kZpAly)DoEa>y5s%N0{Lfi;`wvy53HS=K5A5W_TwnrMP4mu z3(}_hRz4F;;i`*qiMs{WG0W|-o=LWLJ7DGA2(@$=$VD#dee0v{G>KxM zNBU+gG!3^Rc+f+Kq-SFnB8Sx@JD0fDZqA`~lTSZG`VA60kFy{nUX-}kO!ujcj0KLcF z0_Fx|zqcZVa{VzUnT^`tY_!GI&Dnx298%)+>bTe3Qd3c!8yhzy7r!t{`63W-$2f$2 z3^hlB@x5DG;xezMj&HpP%mYJH zpYlRRjx79+@>4y~{)xZj%{|{A-%SSL@?CB6nyd;=)bRs)#1|sI6p3q~l419F_x)R~ zeIho=;{3Wr6EZUl`hG-lOdu;awYke!7(H{A_PP7&E<+OsgwM%4CGpGDj?Y=e!qbOR ztrzpX*oT=;iFQ)golU9bj3%0h)06PE+D*08rQx`xtZUN2W3sRjKX3?Y7#OblPS8O< zWyVrfara9!?-n6UMTq?p0u_T}@%;T>B+}o>tsh_ltgij8mcDdji|R9v){&~DJgb!%koMv8>IWl$vuIu`(l{;F z;!^0VqQ`a3v=D_=ZFDggD)tUTZsGyf~CC7SHek*8SaVjo>%fuZt%CbZW)1r9H)EBDRg$By_jc!m2&Hr*9X(v}7uy9IWL>>_XLl>hj1`z!ukMQX`gJq;-s zCLA^1ZC7lu!{^XLyOP@t;&oM}lbQ2(^?uaYqlN zz{NODgz)Fy%Ia|hdyGvVD}ypVW4p9sOHrI$jsazgkVr_%Zd@oxC{E#IVfg5H z7D}4AZ8oO;c6BBu(*lRFQFJjGg>Q-v-8)D8{$9WW;dN@NA+6y3iBB|&DVrD~qCi`s z{(JcQo`p9dK26*O*+`@;#bvP}t-&V1CV}s)c)&Lk0&_#kP{FZhD#lV?=dwzyT9QV~ z&G(OWRlT9-6KPYg+|M=iCVpluh(1g`iS@xq`HW#*TwWec^EZmT=+hkV!8>rsr_-17 zP5I0aX7*tDFg3HGz-0*wR*GhIpTbtFcz$O{E+uVz3EKVP!&x!01-0{iRVa+ML5Rp^ zQPBJ!kG)l9&c*t_BFmVc>j{a0Io1w8{(icC^~J3na$nA*vQ_k{kqb6f=}Z^vL~((; z?CcP#VWPq_n#7Ie#Tn0uk)mj5%8S{fEW8z)WS-Uhmc~F!C=S6T*6iJP*}xu&xF^9x zoPEfMs89Zo%ria65t|~%|1|uBepcUgeNdsJ)zy~SKjK>$cx|yMU9eTrWE#!^2*Sst zTsHdD&!@3Gp*+59MHl&!br^qCesF$Bj@16{Gf0YJ@dpVhoVM12$v3*Pw>HrXQI;3m zy0o;J9r5sULI)!-R+vqMCySp70J#d2=aP-|>(LE}&D+E3(K{DxM_G*b@4 z`q(Pi;&VslZ4zZxYf6N{!ppg+W@U3Y?piuaEamp0OUvbR6^64W5I4SS%zOCc%hcQ9 zBSPJ9Hit-!IPOce8>{h@4roB+u7ws26JF@Du?g0*^$&bh#N^~)v+`Fu*7Hb9-o0bm z&f2`2sAtErD=+j*J#B0X^I=;2LSe(t}RKy{qowPALmMwlDpf{2g9 zRGbY{iq%b|hBzG!2ze;d{!xb)CdRc+lmwjt>}yg1iC66WgGM20O*;IJQe#?!{_eP6 z-i)3B!TE<^^L$Xm-MnA9027I>>XEaz^n7b_m7#NeRhe6XOEnffM~s`8s_<%M+z)ag zUqt!~-N!_`*N@P69hzG?K#b>z(c3LcO}o#l(A&;2Br57dkz=8NOZlRldyE+6^Yp^f zm{NtqZ0!0LUmbxN_2D`7bh^vR2~6ruHD9m2!vVYTb5LT{oUBR3_&Hiqc7x0E;|ow< zCfAe$mbn?u^Iek3mSNZkV<``%r#z&sJ@)0i?QPxuXL&sIw{Vtjt7i(*?u@zra9wl{ zTHX?W1a{97 z_bP}O^}73-LKybZo7?%1));&8ZTE_cFVa2y%@k#kSpD&s%h?Ds%_dHW&%*8?_1cv$ z>%KMiYQ9(D`1h0q53=e(OW{hTE{PMOts7G4mebx!vh)aeZ|0+xYNw4)FJKFJIw4ym z!KrUfX(56yI0_^JXlZpaC7X)Q-0Uh(EnvUxFZV395Uq^=R`at>9DTK0%o81xf82sh zhx|zho+j-Mq#i~ zKpi79Vj2HqU<8+Uf9}#m+p&a4v!Aw6Deu_FWwaD`t)l#!RF&L0zKToRElIPGGrM9K zexw>*@(Kj!m)|dxj_iKv%Oi}t=*-}ZLo9m16N`B@ajCsn(MKpMUhzVu#Yf32svw{1 zW8fdVNWmB+-9t<9gSlUK9LC=f6?ezs5bxKs9q&Gz`F1r<;`=5OyoTYGF{#yxWa9SLfO~9Z@Se*Ho(R6H?qMoBW_)m}el+)H zsCoc3AH3Ww7!%^D{k)6Vwr-A1i!8W-ER89%JMd?y^wDxzak;~Ut{3O!+U%1`KXeK^ z%fF2~=R@c7F&tOnMb+CFhOz`!9;Vv!^xTDTL%caB8h@BK4Kp(lGed_yjb%2wqWf)X z*US73WxS43t~mI5RDWQ;ira%zxQF!yiDNyb2=1?^A&^AW9H*^6C=#U_&!uxKI7!@i ziWwnKi?(ba>ikf|{AmDPo^zy1-Z};kW%+dR zOBYz=QBOa9;j!00lKh46kP>`)Pw%?8>4l^1;cV&&uMaD|n0j-0RV&{1`kk*jcdqaY z+_@_tkj<)d<&z)sd~CPf<##K=fqXs-dkR?h4u$D?&vp3vZeIUft;;;7AKtfx>L{+= zI;DR%(QVPnf2aNPIu&JJkME;)Jns)W#kC25NRS_@Vqd--lt0rEOOL|>4CTf^^lOonFl#Ll2E4H0Bc*JJ8?Upyl&^`DV*I3Ej->Bxnn6E(BL zyenJ`Z4h#jQfv^3+xM03aj-rUnz;d}_3)xkoCaNs@0AeQ*5?j;do@aniRo%Sa!cu| z&dQt}gJ4C@n`b(N{zr@~Z-igo2HY2xS3 zKRYHDc8_a(HeQ?~5o!Aa5%0^{381>)PzX{#kyut~=w{odn z!=tpx3H@Pt<`C{a*5m|iN~mG_LrX_rQ&ywpNoa#r;++OL#nbJ%)PEfLMn1vM*STT4yooHn>Iyod=Ior!it6>q2`Z6ypJ(*78XhH zwR^LNJuBWhMFnSJXsu-j`>ATOjoaR1y>o(eSZ+vK?2W#p^wEo&Qk|g6z_ud|i?Z@( zL@WmXL^3{EFm`<2M?s<<^&Ai|kU;msVU&s6cZ=-fvcug*+@S3KLA$5@7o&(c+uoDB z5Q!r?tEU*}4ZcuAsB_YtLV0Yw5LY|=7fIJar&T0ey1n=rhwihgAaE&P*&uk&0<&`N z>HZRTn%b+=N~wB?-@5|Z*2b1w1`&k(gZ={{(;x1ftlHRa#X0SU_`y6QVUo0FTN*b6 z`>K+pkrj|sZVxLp`J$2t*X5Ap^IH<7(Z{q6eL~i%CrVZEXVkCvlY)(Yklzo#L>d2f zkCE-}qT&p5Uy=&l2qRg~v(fmOMX8`copZkgMg}Bv6+NV0ci5WgVOc1dBj3@-M?orv z$~9F?mdLFxl?7MJ?SrzB)!C!GYx~zuh(hywXlw^9|BSt_ATFP_6qmfk6@~=~$)KhL z6mw5JB1eS#uamobuO#^;t5>2586iVnB3WNPitJIF?Lpdsqwu@SCRpa#-q&B&5ToC& z$mMV@=P@RtEuzs<3=n*(pD)_SbT8~=!RDTAl;Epg*KPv!bpFmyI+T{Bm+!-^BZ#=k zlOnxZP@*m!EemZF`{Lxr(G2?88t=O7yK7~}wNpAd25xJ~)xjc%Xd&o`-rJYr<7b0J zDsi3PT%ri#P=pkrC&jFEJ zI2$)3N8Xo*+g-fVeo$(ChRhb{BsewYQBj43(|zZ=Br=-Kyn^NwlBcIyY}STwBnRZ) z=(jYP>$#C9t-fE8%~A>M6JuPjGz&=`K)cQp%j4(b3j)Y5Vn%-UlLBxZb?NO8c|?ML zRvx>&mvMiWuIEW~$|bxy!u2SGBXi^RGfKcWZ-u)!zupij4xGzto1czLzV)vfEZz1_ z6D>~4sJ)NJQ^cjeru zG@_m^qostoFn^*p%6FQmneVx%txtYFzTWpqh4|n>)lmfe3?=4LpC@U?{y;HY$A7J^ zUq~{Iuk#i1_cXn(wSzgfoOtx86K#MsN{j^}|HjDo>O;F=8lyV9i|e9isY&qzCHf3` z@c{>OUyTgs7}_vr?2uj*yplf)BO_J*I077`&g6_`Zf4xU_|;d!qwM>o;0XDRPg@nr zq~WW6?w@!%xC3}rHewyuhamX}a$ATD-%E@(zdE4!n2+Bht#UC12QCI96uT7Z6Ew5$ zk0l(DvFr<=wRe8M&@+6RF25EGS06RfBL~BmwHDpf+X;BQ?E18~H#*4|H2g@ReM!F$TPet)p)6ic#4t_0bzP&jaY+C1nwH8jUomtV^8nyu@Y z9%DyM$;<_wUe`EA_k-b7PmE71XI@z~1!Gz0+!b;4fiue1#!s|i#snQXdqRd2E7G!)i2e}xY?@&vA zceH2W<3Sb};*cM0N3Y^%vsB)lWws3I#B!0Efz9>G_L0Eg@~s0maj%_*i{kTrr-h_0 zmEKRAa=9>^6pkH*Ta@Du?ds4kr_$Dwu!f74UP=Yp@^GuK%@Vnlcp2|=IsR7y1)$|H zRiD)YZL`m+*|xhDQW#~a>_kb|C;_>bB0o#a_L74ZUgb5UDWtvE+MiOFxLy@>Z&zno#G6@RK!?K3%ST_JCqw>6iSF6Bq$GvdJMjozK4VIZCAQJFfLtO1f}ay>uU6 z)NiV-Ww`Gl6nv{0qbWzGf<0G5mOkiJORVXvu7*@4!Ed@uNFF@ z*AA}$A=Fh=YvJNl&0{wk?ZI!!PGPmF4Esh$MzRS;0G1Jnq`T64uvaz&_ ztBJm%7pQ0~0=y;P!j$PSH%6Md7)+xgdp1@}jyMlUe_5S%#|&y4-Bs zPP2^@i-|#=N(Q;fo&`K5IqI0_+(xNO8PYF|U*AQ{(@WCvgw=p(K_yun}WvcNcF4g|XP{0r7Zr81$22aNA3u0tGmkap8XG9oEzile7+ET`0G;4Gh&Ta*!qm~a*2UD z=1oG6pZ<^1e7%S%VOEFgA~>^c+ItK9YL<$#*IbW4_j-{0;Q?Jw8}xe}Qs9JTe`NMp z-eOI4L!gCT%%9Ych$W`r!eT(_g2kBjyUqrU{ZraYn$(uQG{}XK9ALzZkpT``gLA87leQQ zBQE9D_m?*y}BmP{GoIe0%B%v{c#PZpRX{VqrJ!-+3BT2jnC!2kg5b5F_ZXxUch_>0wbtXrSDC1APzvQ{Kus5z%#OXjlflPLHFfx$ z@#^GQ%c6P1EFzOpukH|R{wIM>W``B)-*ZiM=Yaka{w^Dn&-eKo3HS0_=DBlt z2*+-(Z%v8gIEDz)4o{KY7g!K#OMN0b@_Y9gpU>2YA5$pmPIraYZT77-tzfOI3yi|B zhU72)4T7(Ye3^{%r)7dlryRHD3VA8Uhw({l4aMH7E4C>s*QN8c(wC=k|eW*ZaxF2 z@OU_hvf;n$Dupd;TeeomA$!$Pa4i-={b8!yl~A(uo4LM1&vFe#lrFV%F@E$^a-!r8 z1u1P*0-rh(k~C@xl~- zV(zS&22CL0EL(-T)S^Qgq=mI`>wXk{&yHIB1i}k<d^TI8x-eMpb4gol^y%fE?K&y4* zO-n2NSi=HUYc3;~?O7=wzI=;z|J~}M9%5qd!Q08lHbL|~s!kbc&b_lciqt_2+)H8i zGDC-Z;ga!nk#&2u&F$U~Ro?6-ci|F|?%5(bWi95qho5b^>=^ZboU*zip0S4#5Q_va z_i|hyY`PX|G-N|zEIl_`VVq$M0bBTE z)MNTuSDAtrcqxYBx4}{9aOf5KsD^nGa(>VF`1jlo# zf~2`WM7Qi@66_^?)234F+0*xZ5<;=*g4qU_?)a|dm>zX?a%r?|(0pj$^cqO?A5?95`PROAK5!SgtfN|O`PY*-x zfBSUHY-iiP5J33F9RSp2&t4=IA4TlqU!6SndOs^4ASqrHoWN>w@3+gFsZibdI{?71 zzw|m;cudra?x>$9J%V3^wO#2<=z-!9fQ?WxluMY*yqTxy6@=u?zbNOye15;$|6br; zwE8|6xs2=Soj)&G&Fk}xpzc3zQT$TU<4d4!xi}oWU+~_yL?vG)zo&MP1NY8dZ2uSoZJCv|mo-cDjH#_`M_o&fTek5fZhutZuj{dwaveOmyF@ky|G`K|Jq zK0b81^wdRzw>-Na$2Fq`ZcUD>HQ-KDuI26lz5^-@ufOU=~ImK2b z_b9^j;2#zQ9a5boJ7g#Kl#ue7u0L;$9V!r=B7UHFxAQ{#(aeDt03MY$$}bBAU&z3= zGQ|o}GCPwHag(0zF$MvOON)HPTQv8ZZCg&(SlxG$``#aphs|AHz-l&cw-Y_^;^ZH< z7B}<2eLMa5e&5+^+-TMaU5a~5$1*qz8dLxEYpIAOu9X8_a))kT{xLirVjs`<@K~u(N+L`Lng1$8eMJ{En~WkZTAv+Kb(n1*0-atPjJ+tgX&`3&>gK0E zZ|x#l=d_~zA;N!rSg!;_gD>PbKiSSN7hZEyoaHGi!yiMF$zs{=P0D_= zZTP{;5|+M6yuJsXkh9bsS;?`kx`eLBhuDPvytS_P<3I181APJQPI7yh3=r_m43Lr@ z(3p8=_+5{UiB8ge@}m#c`m>+uBURpRgV7Pufh$eJhpH4j{XcmKOA^LJL#H3SWu=e$ za-V7wo*^WDzV-kE!ioO8&PQSv51XHF^g_&&hI2fCZ}=Me)R1CX$^j`Q{5Fz;I&ftK zpfgw2&&>ZDa~nX(u89G&Qta>lgaexagwp@)^s2hb!G;WJ~R2ob9bD|LIj1a1k3cXQ`vKLsw;AIS$wS$(wE ztm%R)5Qnb=%%>p$N_m#2Y;Rs06Rk^^{zHNOr_TZYbU0jv*&j%jb=7D~iBhf=06~g7 zCb#q@j-l9dS_8;!pAk?BoGP_(Jil@QS~GCl>jYPOxUSgdF!*stfVNSc1glR&FVAJ< z*C-ys2O$$DQ0rSY&`LoB0j7vPpe_3YfAxHu7_JS!yDypF2g+`&{A6sfEco6Y@i&gr zga38U($C4annU|=AoGCbFj@EuwhMWHP7MQO9b=$g?JlzeL?cPBna*$iK%4rw8kB_2 z17H+vai1LwsL|(E;(_C~U&fnkJh=$G>(&W~Y}(<(mFFwZDgeWF2*}0e<86`d zr+xs6Z-omHZ3KYF`H$*DVsP7lj^IK9Y=s*Clp%ndarNI?K%J^cz5;$sOAOcelN zg|&xw+gUYzkxd>3#SABa0}Su&7-qjF;sm)t&dM+3wF)k+IS-wN*ze*Qp8C!U4In7 zXDeh*v^oP^HDx>q2@D}ajbVZ>h=C5<>W6967T7`TM$M_!H0Qk5YBujSVV zEccn%)!@g}h*b|3hjzDDHT#?M6!)eT_uP*{`N@|zIk|*4>FP?jG2^nX9tNd&)u%JSV^seEj{nuv3Zhve@ zu00xm!~K)22-U8@1l19^^--%Ipv|g*vl#k_GE|<+_>|$z!ce)Vk}ZB@AD?hD=USdC z|KwSb1yz#U<*R}-fP_W@Plzo*TYHg=XNG@d@x|>^s&|rwI~$M5wuO0MGM;4rmg}ln zb{w)wHsIg-+kf`$!I|o@Wk8U+SIRaE;Ojpqi{M`^d~o^dv@VgFr4UBPEKN<}WW`(?$>J?4%cXYN*at}ADRNi8^{fRP5MN`IlB2=uoemn;VFmX0k@3ws* z_kyOMQnEsQ-^W`AV{K2|OOtA_JZzv21|;#pZE`aseE8i8$J&W^*BKEENnL;sMAHVb zD8A74(FPqZz0M&(Q7$f#bn9-`=^=Q7XAoIB;joOg zF>QWb&vTyLt68pmy@B~pIMrV2ah8QQfF`0c-(PAE0rf%dSYDU7oDh?>3`Rpi8NEU? zBAqU@$N>dENv%2~M4<&#{PF{!$W||wdE-Ue~ZtMxYHYdf)|pOi4};L2k*TY z;$=Apug~H2hfOw{C5)UU7`>yU^j!x5E|1SI6W%Odqr0bzi2o`uSIjvi&QuDpjPc>v znWNdp7w*&P6ednFI6x-n3hN+M_dr>)915^}RH9V%S+e23QJtR(9|O}1#|N+FxxF+4 z6MRwE%u-eueC^(Agu>>xD|$JT9qN1eu3Vr!jF_GEq^baD=7hws1bSK&Agq7O*@)dd ze79bwmAw_pFK77q12}*Ql3)5;3VhS-sFjqQ@)1emGZos%U6gj!t*4?~<2ls(;s>q` zOyN=%PnN$rhA2MYSc%24aZ>X-filbU-xa(ng1h8JfEVPJy z+!@(!=|}VS_2)dAd1o@*+pgROh)n2jJ%*2|E@VTQ@Y|moEQ{_%)!x#&+(sT0L2G%b zQP1QY(R0?8s?dLkKdqYYl{1!+P?Njd)dfm`Zt1yZ4Six%;VcE`T&H5lsl8GV^lyM- zXD(F?`~FU#Kt+FVf4ReX*>cwJ;my8)`_$(-nR?KfP1$l0!)8Dc{Z9`FIaJ&Bl_ayS zo~uRCY&Ii#rYobFKT|z%N_ojX61k5J%4fwWf^_s42)e&710(_U%#}U22F$u-9R~lw zqlko6!z``TdnRcIKgmRuX=xS#&}pTExVYX^AbxUPd-O#19sE31C=9+0BJ|FA6o)`b z!v%>_Mkpo~T@>;@uIy&i$%3v@>)w+>_tIL50J=1vP&g&!fV3>K|7lk=irCeV4>hLV zCq1?+DQxgsYJV1;_5#iDn&B%tON;>#yhDwhgBWn(&q;?^hnC-r4zE5;xDabP4mtQK zBH01h%&3A<{eLgrRvw%Q^z98SJ{i_#r|Z&bTK^~^VI)n1Gis{mEo+FGGTmKbgyTEQ zBHzy=xd9*jEy5ZejdAY;UUQO|(PD_^RSVY5d|h&{hz|1#v=@0;2c{j~l_BY%t?KQT z-J_OlRj=&ZHtC={U{npux7W#m2pq=<^a|X`@ewgJy}0Sk0bT5JWI)^!$>^>QG2E06 zOW4ay&FSLQUfcD0YH!XxQiW2d(`U(SD1%R+Q(~)1mn_wK0 za=bMo+m_SR?dcBKn`Cr(!)xJd8OI<|MLn;3XVfj!R(ZWs3Ol%jG<1O*}^4X)To1SYKIEUwX z;dKt;AK^@5+(+ZtndPf)E$jlA>+q0pe3X7$%WmUtd}su==wMhC*m7Jmh^Y7*v_JXq zCZpOuY<9&xzX&WQ^-y2$c`cHF(PkDYf%~8cG<4hU9dFZEA*-S9lh6LLl+m9zyg3(S zr`NmIsF{}d*&+|#WL^|!6_S|57I6s#m2CcJ(|A0C-O~s19ZHc4>7=ri9AsAk-L=6L zV2q6m?sHcJHaJT9JnJvK|IIU22jJ*u$ZtjVJr`xcf|Y2wwRVE8V&4frRp&Pxeb4t5 z&js1P<#9kna20?MA?9cQN!R_~2S@fJeR2x_(N_QP@B&DK_|Goy|NX#~-}z4awl{J^ zV%5x%Uv`KNlm5E&hz$)$-NEZl5VaKI6U?T|vIsAV=)-8ni;`+5DGoX34D zU}BCF5*~T=Oa7Ps3IMCCRMmaeD4SC)gJ6~X?_D_pgz|uDov{zA{^!s6_rD=-&sN2^ zE!Z>q`LD13m%mrq^_)CH8l^g@$p+D_SWCL`OAYfnd;h0JSb^2GUrMthdMWNw-u#JQ zH?U*|oPU8w#pFR4`hW2`AUi=DwiRU?EO!$<@BErZ{v|N}?#DCh*H#7n9`pvt5I;{h z{hD2$ssRjlOW;!gvaYW6^MHk7AYy}O=zsP3JQ+xYuMoK#O z*YN+-;H=X@xQDasSkPI28;X4=B!B5IC9~C)o4##{jFXW#g_3g*Aoh1zq{Us2u}+vB zMTp*BuGjW%OEero6o%_Z1t1`Riv| zF0vw!*#6k7MVU5g75yjcoH(u^9iUrG#aE`y>=sN+9U09N951NyS*x4)6y+8;K8Cth zb-7RmT9HZXul3|Umv~dWt7li)7kMnq1DCUt^{8@=r2M+%hqK};80*!GiQ+Ergp97? zVDkem_>MyBe1bPB4IwmP=1Q1%$aU}fyguf?I8br{Km*7Ud6tOZDPpRe?Wl7InZ8Zt zDy)X|PM-V&DWCO;3iuBXL7#^El8&2JTN8+USm4{fzKlRk0+1yQZ)GZOeiVwMC!mSCzQDGYNa{ve(o{YioNZ?Do)PA^(xdZw{EljWJhoZ+{a<;wqB1uZ(gr9)^Ne| zd-X+L^Yue>5|2W8-9~TqeY(Y)3o}#C^|3yfs7Mp=U!^v$e_zpDsJ#BeSt})aeYi)= zbo8p(r*w5L*#-^8(wD^5QA-y^^5-qfxs$ni(;kAl1tId=qnBy4_ZsqzWHW72SOZY? z5F!f$i;gr`b~QT96Q7xyB$UrunFnvgGkWhtL|05nc~|)1+Dlwe!!sS~*Z|7x?D~)C zlz|2*MN~)_gAuAC3hFj5W|!{RxRVxc_>9M7aH)&sZbRA-J_qz$W4QNvlc(y!w=_ig z5y~kCTGVN$en}_F?S-bTMUQ$9uwk7CCR;U2SVtA7jb4MA`6aojmm(61YQAdiXwSZ1 zPu)%Z169ljve849iJsM1LiT%HF<8}Bx_K#;CN2z2v{t!sGMVfIEue(u%!nUfL~(6B zU#MQJg*CqCM2||)@c4pk0?N76A@g0u2b7mL=8e*(cd=MUflaJ=&o_*McXyyV3DU8*^RG7qFEq>Xf|*X`NZ8r&E*u2`@MJQC%VqFZRcgVuBY zo)-&Uyx6E;igxgm*923P-}lyzUvt*)u5{oinq!^(P_ZS*u&#p)bx($BRTtoEX08kB zZ!g`e1WAG%Yr=WZ{`~-&?CiJOE?IMQw&!VgZQ`(orV1o^#Fe|gH**6*01!L8hp=umA;oo?csb8{vSv8Z!q|2VyNZHQg> z5(p*2idg!c%5R4;6242Bm!d`tA3OF3EQI&(uBR)uY^S2;Nj56CLRF zlwOK1rXUszg9zNF#zf&9|Ic#*_=?q(Il zzvi+gqcf;6?e@VX(&ECPQIAtg!^ZkZ4$73#AZYz<;#RCS$jcuq+sv8b(iirB&hr?) zc}lN4$!L&W(?cFEB~d&4K^Wd5T)XCw>-E5TYa=tt#m?Rf2+3UIc#OG2RhSh613Dhl zra?~N6mL%EwE;-t_7mm>5}vs_d8a9@Aa*q`XPOXio8lP3Yt}&F8fXl|mOIwEH>$=K zZ1z3i_45iJTP_;EFu$GP?bM;$vru(6cUVq?u2VmXTxC)5`9bE6=>pmSy@3Gf&l_iL zeFlNW)&>{p^H*dg`|N81D0^Kad6SG9FFKzd^dSZP2={b|;TD3tHS+D$>@%k>B@F4hl)bkd~XIIf~xdG zaKm!-%tdnGA-o>}+zm3I#;VRprh9^Oxj9%&T()LCr|#rEdO+D82v zx>gtVd)6UgmmB(1~J^Z4JjvAwKdaE9wg(7s~B^K;4EBYkium=XG-M~dMZ(5A>*q-rQhoz zVNA(W=9&ow(ABB1nQzN6VL`@{d5(+uaKg!2YE`1hV6omo6rb}Eh*uF&V*Pk7AQVz? zT{gy55>h;4qEc{yO#nW7c@o+4m%Xus^_#OV6Ra+XXiWGy0nhdIu%>ZE&Zr9(QOq9& zqkd~2krb*dUB)uQ+T5*ah6BpD+uHzAL_8DKZZKVR1?z}>KBdYr;IvXS7bc`nr0kqw@ElzabY=BB%2D=g` z!1(VgcQis}%ent%W64RWXfSjw3U4G!kD+x@-NqcYw$RYw1QkgfA+5iOaR?{K&rZ#R(iD#E4zpH^C@g zTF&-#__Y-H?SLrIuv^|-`6;x&XNx7N8!S%jTQ1!2!2Up@%B;`!T&hc~U;f*QejBW+ znd~jbd#^TFCY{l>mb$fg4ilZN|nv73I6SUfMcX;^-!LOO2ihS!W1oR{73 z$iMX?fW32l&1ouB55@j8%4kB0Ta|+oK)?e&4{S7#9wdofdhu72uqJ{njN~PA(p}$I zN=(zbSuKP=oxz>??`|O(U8~EN1s1&M((18_FL*Zu){Jo z>R|lU{5Su#Df<+q0sb}YuV_Kj#cQ?G?8TTEA3woPmtN+L5mT%63LBJm<8#E7O?hv2^ffXTp5}>57#zBI92j zcryz`4^pqF4D%@YzB(uX{_Hvb2;G=yQ*)~QzXmMp70^jp-;dM2vO*?qJ^+9A__@z5 zUnX~VJMMq=PUr)0DLEcZ7*X&Wum2ysl!Tly_pJ)&iYbCfzSG8Ar@b8-;+fTr{bsU- z+Hw49Ce1TlW6O0seS?a7+m6`N-fH1D9P9Qvb~k8zvbzJ;BHi-UL-8@RNH)Y9X&r+p@rM$QsZ8Xz$zOnQs68@A{-tp>j(_)7c%N zl2bN1$z6#~2$>>hBeXe;DL3g(EOkp5ZiJXQZ*!VsStnG?Y36(=Hk-qS4f|fJtZ%-l=Vp0DiZzx@U=I(z!W)f9iXa_+X)_f-P_Q&rdh z1WV;m1PmGt>>_FX%CN-(OcG_L&I25flH;wu|I%`N%k%|mi8`Gxywg42h6a!vs3<%N zqvp=YtJtwvJ+UCf2$O)SV`EQHaA8u=*f!K#IWECF62${&ZYIF%&4xJia;vCFbz&O} z-JulXK|JgyUEmmLM*?1>4MjD-IPn;d6lIu8mL}vCi`*R%EI?V)<2Xty)s>EYlL)rc zbVDrV2l{b*N^;29e<8TVE3>CuPXp?pFYI(Bs+Ydu-x+r{dZ>6~Pzh5*Li7UdFrH7T zS0(X7gBiqGXj|Rn2EX_o(><3C!o8}Wr{}s9&NFKx*kqubSl48b{t)R&Z&a~(Sz>$> zHPP*gIHpZj9kWH`SCKyUSeJFHnCo_~=55=&&67XeZrAnCHXsU_v7XSnDI>Q_k488z zn6P+xV;8xaY*FRv7qt!WAh>0={_qR#w=e9$2Vm6v)IY(nzVPP=O(&R>Z5NU}onw2W zmtgDQ7Y%V@2hZgxc+yKQ`Bs!P4U=5nu2eW`fA_$l6kjJf1s(n_)k~G9li^pvTygHh zpxVI1rBH&hlkAG4X=O}nv}2DZ$zxJA23~z4_*j>(S#v?6KCftC-?^@^!ad9xO?(GT z#=t3|{aP0P>_kK%uyJ{{6{@Uks#4*l!OTuc<>RO--@A;CGy;;X{~^<~ym>-N>y_u%J9yicg80}2e)ftg zF*+3hBaKo;{|JKL8-10-9Yzel(pe~m28T6Ht3-}}za=uRt%TAUnB(PmDZ(6;~h zXGQ7>SphTBjP=Cbt*qeMY}$E5Q>8PrQSLqB>%uT;hPy9@GqeAHAvQAMaASUn?Xi4IMM-ZFLN-bAfPdL>$8Rta->vCPJDRJ^jGO^C+VaHVa7OTi{ zjl#(*Da0-?UehO_|@k9(ZR14xmhV8_Q-q!Vm7+8my6+55)uX7XO%7P zP zRs=u7+;|bS!?4SO!b_Di%qaDS}++Qr49T);Jn4r zKO1RuzK(+z>9y%G(6H2PQ@@heNMsUDT}Wm8IpAlvA!B5q9XA!;51!uRO-C9 z1~PR2rRi35mO^GZHbW|j(B((2CV44C`Tm71PRj=7PT6}4Yq7$GX{?%cc?}${+Y7gz zOG7L=;F04WeX%$3&hXn0X%%xv6tr)ge(`!ts&&uE8EFLvxn^}|WlJcnNtgBLsPoml z50b7c)8HFG?%!x^9GumYFa5lw3HG*&G~uGQom`YdM6r$Q?X_~v5#80tR{#ZpEfS3W z|FFB<{XISx#3xd-=O5R)6H-QW{2nWgw-susY9TjUjZIloN_OE69@@HLoThGNB zbg5U9ViZW_96~c=K-ZFH&;r10dSDMKB2m7xTF|KN^;KZ8^osE`x_XYeryZaD`i^$4 z4RF0C?Xwz}{gmS?>j!~MEdu1*qPWBD<;hTqQXqHXS)7vS$gY|!DP>9`-piCIRKYNhh?0%V+4l-w9pqyNNqsoPu$Q zv$dp3P#Qr-zPXuQd~t{ALA%;u-hB(H-Tj4^-t&3W8ivMQX{k9;)^O(?USl2#Aw6G4 z)ZJXhN|ZA@?%oV$q@QWMp9fBzP)j?KOfSEi2g3H`UduT35>Qltvc7bK4VOdx6ai{< z*QciMzs6m!QRMn`f_y+GiYVDK(&6fibD){3BUlw&zY{*(cf)N2go6lb*13ut#46-Y z1jI6*yw4{_gOt7s)!veO6gr(TYv%2%n4U7N%OrVfv-$;$g*T@n<;c@!h7FB)W}Dy3 z#i^Dv3Ws0Du`K_+3lqFsrxQj*#magB`~hXOsKB`Xp}_OtbJ@0cAJu=X$hC>bF*8uO z@8B>4GnHnP_uKd=fm^Tj{$DAN8` zQ@X_=^Nr>jFSayZ`i-Vozq8bc4l@4Me>RCf#E?G5xuIa-_JAT-Qn6VWP~i0golU%S zV>k4Jrm0~|a=f5qJ<-|Rfa+u#&)PEn6=|c>uAh^9UX7H_%dk6Sd~*&QKHk3D?Se7! zHZ-E6AhWINg0lOM)Ma_~#G`JW!4 zC7+&(vZNK5tM(Mc$e0j85JxfM+)$%B@a+cQ!GHL6JLgwT?SYO>ZeM2ts_bP3M3QH> zxzU>RbU;b5zHwi)8Jb_l0MT6qVXE2GTnu}?{1IgXIw&E6oXRTs**NrRKq0&pTEWmubjiCm9kt zgceA?J5`!qu!Pnhdp=mqjk*uvPBvPmSk;Dxn@oc;EWEysY9SB5V<9E2v&&iW+%p2O zcZ*-YPFxmNE|HJm{y$c{mT0_K{R zLP(bj72=z_JT*P`)9_Wa-0Tc;ju-p9aT1dEn_N^m9tF1pMj9$y}?7hKKSRp2pA439Ja4@%@!(cQi@*pVx_!SnrQB`=C7r z153I%#271-^E^;%^wDxxombAwL44cFaUm+lpl&B{R~c)u-YK!61+O6V?l&!H# z^@F!b*<`>X9;4wje}H=TaFTo7fyPIx3Q>h_`xm<~(yi_7Hq*^Pz>1344NTsK{@zF7(^Ra9DW6?&-^^Bu^^4f#%Fzr0`gQ< zN}qv%=^%LC2@piou%LD@__zoltS1`QIh2l_a9_4iSFE}$A`n<}d3+C1*=uZrIEb;< zU;ViMOy%9|)*w(P&pgrhC)0MV2qp87ONF83Ky2Gv9CQXLRJy=e8(`}BXaj_)bJZYg z`D6vl=#>bFGd4lRlEDmDavjHh?~Y( z?&$HHwSPb@N|{Ml^%(!?H#d(YIf7lHik>ii&UvnHXduRp_u|0-S*cbCDgkC@+Y8kL zbcW=DA~AGagI-aWKD+P33_zm(FaO^QQduRZg-pN*@oL&62J4i-T?SVVj3Qo)9v(#< z5S*=Zc=s$>)_B=IKEjT-uf@HnI=L2m*0!foPp{6Qr?kj&*&eL%TM-Be&c89jQWmAk zP7i_S2h_bY6+fR1=B_91&~&mRgcj2+(G*#{2Rah2A`r8dMARIAw$fE0>4n@tT!?Z@~5v=r?RvKPs~9^`|yL ziJ`WBRtWdrw0mEBxGfshM8SSTSB--~jA#|}I=5u>Ge*ziet-!4o54QZO~+Khi<_*G zO7>6L+incMR~($#Z#aolLknyZec1oPc+;N;hIwzE`Xat2f`d0N+p@*eEhvgoa3dbM zP4`{Z{6sYh6Z=lveIN&(^7-j&19CQ%`hQ?{$)iK2#-yaSAe?U?WW5 z5%lpM%00rVt@`|U*!8<%)UX0U@Ws0HUT!*9JilZ~gO1JFVT#+5ld3za=9~zGAfar_sPPQ@7CCc0wt9*5RvLem z-oWt2D>x~5AibDFOIKD{_7_fSAsM5xWt&9#>R&;=dh(WNC-G?!aPaX|m}a-NtdbH5#c&@Ve_r8Da-Tq*IbC(1zJIJMW7bZb5b28_~z^`NvbMxAd#DD>b;U_~t zUbkz%i?(=!QAGoCiVF4kkFjY%l#+Tuy^MZqlH;H>x${#7wxEc*W7tA_kwa5v4XqpQ zcK=j+FtEwNN#GjD$Uylt-skag5d^+oR>dOD&)i*rXn6*!OOK}9?p$Xx&AYC8`9*=k z{lMd$(?^&A3xJ$X_~=c`wG=p^W)VpYjMfLF&><(`{<|K-O7eKXUMP@>2C+McT^-Y) z2b_BTkwu2fn`Wn;XwMi5-J;A(`;Os5tkfzff=1Fp(ky{R@arlOtfr?L2^}ntvYndE zb~wP%$QfP)lpx#xWIPO% zZ(ZP0Bs4vGgJ+4g>B{jVS9;orM5&2@$VPvsDy{LL3IfisQ6(iPP3x;}4Pmo>NJB8n zzS9B0tT($R+$2*DWUmOj`@K_wP5z3J#6p&mcR~ZN9Ze9E4Tt_Zo>rG`is-sBof{BG zXIMS^1OCBUyRvTL3%zz}an`P)(n^OMm`y4+%fDsWPWc$FN$|A(`szpw=XodQ_mvS0 z{m#|h6v1Io@Az^pA&mdY*^a#lQqAX~D{`$Y1q0*kZZyrk3{bPnew!bdDOBH?jPZ7$ z7jb1ksIkoAws4WKbb-Pmm@A#hnW^FQw51MnCz{6snB6(4<4Gn%4F$B*W9UGSy7a8@ z^iJ`XhOFOv1Ul9VBuw5{*@4eMh>5FwTN;@tT&#uVrQdm5&#)6tjk728uxobs*FaR_ zSE6A)bsjN1K2tVKSOqJ{sbYCn4BR3uo=99pZc(Y?8HG; zWpS);$QoE^?sXU0zrk;I<0vM^mI zK2M3@!R=wGJuCVTw+>4>53a@b=dLIGBzLF*GUy$NDc+pOEW7ZI0?JMo<>G!9?h1JsM)UV;R1@)7mhS0+H8bmnI^dbLz5V<>1>^J3|aT-nzfki zv-9yGaDUyG>EW6%YS2xdaOaO0b@}M9M@K>JxmXoP5gh>0Q`rf~g#AKEpVi9~_1nra z@Ny#whg$W4t5FW^ckO<#X^%(E-4!H|e5pUz_ZI?WUrqF9p1QxGuqs$^KP$Cl)$)5E z-+2>kxv)>8@9#-s=j(8aYYtp~@c_|V1@k!)zE1Ak^yrZ8`KFaWFV6y(eZzy zA=eqH(geD<`;O~eM}p9nA0;}NKO<>GcUiSOE_aU)07L2T{1b2zn4q}wv!(2RXJg>h zy$98^ewr2{9gg!gk3(=GL5XleKy5b_qy>H+-%1()(jd!DPc8WARq{KrGG_iC9Qtn2 zDfe?pt}P9sOi;iIFd?-J2K_YIeRcuDsYLP`rG3!Ai+dBq{M?;)@5+d@4c@9 z#vVaye`)YEwNZ^e0}Ss&tA~MO$tilCHAfLho>>(AvQEcKYULSxP-p18X|ta zpQyDJgmE)1D0()(cJ6;{mxqNB`%Q5!URCp%7vhh+-cfla%M`^dq*^y8!!#Fsm&lKQ z5{|*@KYj!dwPD$w6)2WR!=r5^z~5*W6`ZyPd2KAf_HW`eC*!s>ifwAq;F*DmOaA?# zlbE>tZ5MslRY`FWpVZ%4lYz|&JMjCj4_NhGdc=S#X8(Mf6X+)J)xZ2l!cUfjDAfN$ zU?y7Z^L+CH{dWVO@x{fXVvfeQ&)g3U7Da_$77yn%;5(d6b=fc8P$GN$hfIl{_(1>0 zE9mmU>;K0AegHbrb^g~*tXQ>qFDHb1Q{cFjpV9pr_&aN4dOGjqjnMxBk+S*w diff --git a/mediapipe/calculators/ovms/modelapiovmsadapter.cc b/mediapipe/calculators/ovms/modelapiovmsadapter.cc deleted file mode 100644 index 1460771613..0000000000 --- a/mediapipe/calculators/ovms/modelapiovmsadapter.cc +++ /dev/null @@ -1,310 +0,0 @@ -//***************************************************************************** -// Copyright 2023 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#include "modelapiovmsadapter.hpp" - -#include -#include -#include -#include -#include -#include -#include - -#include - -#include "ovms.h" // NOLINT -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wdeprecated-declarations" -#include "mediapipe/framework/port/logging.h" -#pragma GCC diagnostic pop -// here we need to decide if we have several calculators (1 for OVMS repository, 1-N inside mediapipe) -// for the one inside OVMS repo it makes sense to reuse code from ovms lib -namespace mediapipe { - -using std::endl; - - -#define THROW_IF_CIRCULAR_ERR(C_API_CALL) \ - { \ - auto* fatalErr = C_API_CALL; \ - if (fatalErr != nullptr) { \ - std::runtime_error exc("Getting status details circular error"); \ - throw exc; \ - } \ - } - -#define ASSERT_CAPI_STATUS_NULL(C_API_CALL) \ - { \ - auto* err = C_API_CALL; \ - if (err != nullptr) { \ - uint32_t code = 0; \ - const char* msg = nullptr; \ - THROW_IF_CIRCULAR_ERR(OVMS_StatusCode(err, &code)); \ - THROW_IF_CIRCULAR_ERR(OVMS_StatusDetails(err, &msg)); \ - LOG(INFO) << "Error encountred in OVMSCalculator:" << msg << " code: " << code; \ - std::runtime_error exc(msg); \ - OVMS_StatusDelete(err); \ - throw exc; \ - } \ - } -#define CREATE_GUARD(GUARD_NAME, CAPI_TYPE, CAPI_PTR) \ - std::unique_ptr GUARD_NAME(CAPI_PTR, &(CAPI_TYPE##Delete)); - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunused-function" -using InferenceOutput = std::map; -using InferenceInput = std::map; -// TODO -// * why std::map -// * no ret code from infer() -// * no ret code from load() -namespace ovms { -static OVMS_DataType OVPrecision2CAPI(ov::element::Type_t datatype); -static ov::element::Type_t CAPI2OVPrecision(OVMS_DataType datatype); -static ov::Tensor makeOvTensorO(OVMS_DataType datatype, const int64_t* shape, size_t dimCount, const void* voutputData, size_t bytesize); - -OVMSInferenceAdapter::OVMSInferenceAdapter(const std::string& servableName, uint32_t servableVersion, OVMS_Server* cserver) : - servableName(servableName), - servableVersion(servableVersion) { - if (nullptr != cserver) { - this->cserver = cserver; - } else { - OVMS_ServerNew(&this->cserver); - } -} - -OVMSInferenceAdapter::~OVMSInferenceAdapter() { - LOG(INFO) << "OVMSAdapter destr"; -} - -InferenceOutput OVMSInferenceAdapter::infer(const InferenceInput& input) { - ///////////////////// - // PREPARE REQUEST - ///////////////////// - OVMS_InferenceRequest* request{nullptr}; - ASSERT_CAPI_STATUS_NULL(OVMS_InferenceRequestNew(&request, cserver, servableName.c_str(), servableVersion)); - CREATE_GUARD(requestGuard, OVMS_InferenceRequest, request); - - InferenceOutput output; - OVMS_Status* status{nullptr}; - // PREPARE EACH INPUT - // extract single tensor - for (const auto& [name, input_tensor] : input) { - // TODO validate existence of tag key in map - // or handle inference when there is no need for mapping - const char* realInputName = name.c_str(); -#if 0 - const float* input_tensor_access = reinterpret_cast(input_tensor.data()); - std::stringstream ss; - ss << " Adapter received tensor: [ "; - for (int x = 0; x < 10; ++x) { - ss << input_tensor_access[x] << " "; - } - ss << " ]"; - LOG(INFO) << ss.str(); -#endif - const auto& ovinputShape = input_tensor.get_shape(); - std::vector inputShape{ovinputShape.begin(), ovinputShape.end()}; // TODO error handling shape conversion - OVMS_DataType inputDataType = OVPrecision2CAPI(input_tensor.get_element_type()); - ASSERT_CAPI_STATUS_NULL(OVMS_InferenceRequestAddInput(request, realInputName, inputDataType, inputShape.data(), inputShape.size())); // TODO retcode - const uint32_t NOT_USED_NUM = 0; - // TODO handle hardcoded buffertype, notUsedNum additional options? side packets? - ASSERT_CAPI_STATUS_NULL(OVMS_InferenceRequestInputSetData(request, - realInputName, - reinterpret_cast(input_tensor.data()), - input_tensor.get_byte_size(), - OVMS_BUFFERTYPE_CPU, - NOT_USED_NUM)); // TODO retcode - } - ////////////////// - // INFERENCE - ////////////////// - OVMS_InferenceResponse* response = nullptr; - status = OVMS_Inference(cserver, request, &response); - if (nullptr != status) { - uint32_t code = 0; - const char* msg = nullptr; - THROW_IF_CIRCULAR_ERR(OVMS_StatusCode(status, &code)); - THROW_IF_CIRCULAR_ERR(OVMS_StatusDetails(status, &msg)); - std::stringstream ss; - ss << "Inference in OVMSAdapter failed: "; - ss << msg << " code: " << code; - LOG(INFO) << ss.str(); - OVMS_StatusDelete(status); - return output; - } - CREATE_GUARD(responseGuard, OVMS_InferenceResponse, response); - // verify GetOutputCount - uint32_t outputCount = 42; - ASSERT_CAPI_STATUS_NULL(OVMS_InferenceResponseOutputCount(response, &outputCount)); - uint32_t parameterCount = 42; - ASSERT_CAPI_STATUS_NULL(OVMS_InferenceResponseParameterCount(response, ¶meterCount)); - // TODO handle output filtering. Graph definition could suggest - // that we are not interested in all outputs from OVMS Inference - const void* voutputData; - size_t bytesize = 42; - OVMS_DataType datatype = (OVMS_DataType)199; - const int64_t* shape{nullptr}; - size_t dimCount = 42; - OVMS_BufferType bufferType = (OVMS_BufferType)199; - uint32_t deviceId = 42; - const char* outputName{nullptr}; - for (size_t i = 0; i < outputCount; ++i) { - ASSERT_CAPI_STATUS_NULL(OVMS_InferenceResponseOutput(response, i, &outputName, &datatype, &shape, &dimCount, &voutputData, &bytesize, &bufferType, &deviceId)); - output[outputName] = makeOvTensorO(datatype, shape, dimCount, voutputData, bytesize); // TODO optimize FIXME - } - return output; -} -void OVMSInferenceAdapter::loadModel(const std::shared_ptr& model, ov::Core& core, - const std::string& device, const ov::AnyMap& compilationConfig) { - // no need to load but we need to extract metadata - OVMS_ServableMetadata* servableMetadata = nullptr; - ASSERT_CAPI_STATUS_NULL(OVMS_GetServableMetadata(cserver, servableName.c_str(), servableVersion, &servableMetadata)); - uint32_t inputCount = 0; - uint32_t outputCount = 0; - // TODO ensure Metadata object removal in all paths - ASSERT_CAPI_STATUS_NULL(OVMS_ServableMetadataInputCount(servableMetadata, &inputCount)); - ASSERT_CAPI_STATUS_NULL(OVMS_ServableMetadataOutputCount(servableMetadata, &outputCount)); - - uint32_t id = 0; - OVMS_DataType datatype = (OVMS_DataType)199; - int64_t* shapeMin{nullptr}; - int64_t* shapeMax{nullptr}; - size_t dimCount = 42; - const char* tensorName{nullptr}; - for (id = 0; id < inputCount; ++id) { - ASSERT_CAPI_STATUS_NULL(OVMS_ServableMetadataInput(servableMetadata, id, &tensorName, &datatype, &dimCount, &shapeMin, &shapeMax)); - inputNames.emplace_back(tensorName); - shape_min_max_t inputMinMax; - for (size_t i = 0; i < dimCount; ++i) { - // TODO test adapter dynamic shapes - inputMinMax.first.emplace_back(shapeMin[i]); - inputMinMax.second.emplace_back(shapeMax[i]); - } - this->inShapesMinMaxes.insert({tensorName, std::move(inputMinMax)}); - } - for (id = 0; id < outputCount; ++id) { - ASSERT_CAPI_STATUS_NULL(OVMS_ServableMetadataOutput(servableMetadata, id, &tensorName, &datatype, &dimCount, &shapeMin, &shapeMax)); - outputNames.emplace_back(tensorName); - } - const ov::AnyMap* servableMetadataRtInfo; - ASSERT_CAPI_STATUS_NULL(OVMS_ServableMetadataInfo(servableMetadata, reinterpret_cast(&servableMetadataRtInfo))); - this->modelConfig = *servableMetadataRtInfo; - OVMS_ServableMetadataDelete(servableMetadata); -} - -ov::PartialShape OVMSInferenceAdapter::getInputShape(const std::string& inputName) const { - auto it = inShapesMinMaxes.find(inputName); - if (it == inShapesMinMaxes.end()) { - LOG(INFO) << "Could not find input:" << inputName; - throw std::runtime_error(std::string("Adapter could not find input:") + inputName); - } - - ov::PartialShape ovShape; - const auto& [minBorder, maxBorder] = it->second; - ovShape.reserve(minBorder.size()); - for (size_t i = 0; i < minBorder.size(); ++i) { - ovShape.emplace_back(ov::Dimension{minBorder[i], maxBorder[i]}); - } - return ovShape; -} - -std::vector OVMSInferenceAdapter::getInputNames() const { return inputNames; } - -std::vector OVMSInferenceAdapter::getOutputNames() const { return outputNames; } - -const ov::AnyMap& OVMSInferenceAdapter::getModelConfig() const { - return modelConfig; -} - -static OVMS_DataType OVPrecision2CAPI(ov::element::Type_t datatype) { - static std::unordered_map precisionMap{ - {ov::element::Type_t::f64, OVMS_DATATYPE_FP64}, - {ov::element::Type_t::f32, OVMS_DATATYPE_FP32}, - {ov::element::Type_t::f16, OVMS_DATATYPE_FP16}, - {ov::element::Type_t::i64, OVMS_DATATYPE_I64}, - {ov::element::Type_t::i32, OVMS_DATATYPE_I32}, - {ov::element::Type_t::i16, OVMS_DATATYPE_I16}, - {ov::element::Type_t::i8, OVMS_DATATYPE_I8}, - {ov::element::Type_t::i4, OVMS_DATATYPE_I4}, - {ov::element::Type_t::u64, OVMS_DATATYPE_U64}, - {ov::element::Type_t::u32, OVMS_DATATYPE_U32}, - {ov::element::Type_t::u16, OVMS_DATATYPE_U16}, - {ov::element::Type_t::u8, OVMS_DATATYPE_U8}, - {ov::element::Type_t::u4, OVMS_DATATYPE_U4}, - {ov::element::Type_t::u1, OVMS_DATATYPE_U1}, - {ov::element::Type_t::boolean, OVMS_DATATYPE_BOOL}, - {ov::element::Type_t::bf16, OVMS_DATATYPE_BF16}, - {ov::element::Type_t::undefined, OVMS_DATATYPE_UNDEFINED}, - {ov::element::Type_t::dynamic, OVMS_DATATYPE_DYNAMIC} - // {ov::element::Type_t::, OVMS_DATATYPE_MIXEDMIXED}, - // {ov::element::Type_t::, OVMS_DATATYPE_Q78Q78}, - // {ov::element::Type_t::, OVMS_DATATYPE_BINBIN}, - // {ov::element::Type_t::, OVMS_DATATYPE_CUSTOMCUSTOM - }; - auto it = precisionMap.find(datatype); - if (it == precisionMap.end()) { - return OVMS_DATATYPE_UNDEFINED; - } - return it->second; -} - -static ov::element::Type_t CAPI2OVPrecision(OVMS_DataType datatype) { - static std::unordered_map precisionMap{ - {OVMS_DATATYPE_FP64, ov::element::Type_t::f64}, - {OVMS_DATATYPE_FP32, ov::element::Type_t::f32}, - {OVMS_DATATYPE_FP16, ov::element::Type_t::f16}, - {OVMS_DATATYPE_I64, ov::element::Type_t::i64}, - {OVMS_DATATYPE_I32, ov::element::Type_t::i32}, - {OVMS_DATATYPE_I16, ov::element::Type_t::i16}, - {OVMS_DATATYPE_I8, ov::element::Type_t::i8}, - {OVMS_DATATYPE_I4, ov::element::Type_t::i4}, - {OVMS_DATATYPE_U64, ov::element::Type_t::u64}, - {OVMS_DATATYPE_U32, ov::element::Type_t::u32}, - {OVMS_DATATYPE_U16, ov::element::Type_t::u16}, - {OVMS_DATATYPE_U8, ov::element::Type_t::u8}, - {OVMS_DATATYPE_U4, ov::element::Type_t::u4}, - {OVMS_DATATYPE_U1, ov::element::Type_t::u1}, - {OVMS_DATATYPE_BOOL, ov::element::Type_t::boolean}, - {OVMS_DATATYPE_BF16, ov::element::Type_t::bf16}, - {OVMS_DATATYPE_UNDEFINED, ov::element::Type_t::undefined}, - {OVMS_DATATYPE_DYNAMIC, ov::element::Type_t::dynamic} - // {OVMS_DATATYPE_MIXED, ov::element::Type_t::MIXED}, - // {OVMS_DATATYPE_Q78, ov::element::Type_t::Q78}, - // {OVMS_DATATYPE_BIN, ov::element::Type_t::BIN}, - // {OVMS_DATATYPE_CUSTOM, ov::element::Type_t::CUSTOM - }; - auto it = precisionMap.find(datatype); - if (it == precisionMap.end()) { - return ov::element::Type_t::undefined; - } - return it->second; -} - -static ov::Tensor makeOvTensorO(OVMS_DataType datatype, const int64_t* shape, size_t dimCount, const void* voutputData, size_t bytesize) { - ov::Shape ovShape; - for (size_t i = 0; i < dimCount; ++i) { - ovShape.push_back(shape[i]); - } - // here we make copy of underlying OVMS repsonse tensor - ov::Tensor output(CAPI2OVPrecision(datatype), ovShape); - std::memcpy(output.data(), voutputData, bytesize); - return output; -} - -#pragma GCC diagnostic pop -} // namespace ovms -} // namespace mediapipe diff --git a/mediapipe/calculators/ovms/modelapiovmsadapter.hpp b/mediapipe/calculators/ovms/modelapiovmsadapter.hpp deleted file mode 100644 index 3c677324ff..0000000000 --- a/mediapipe/calculators/ovms/modelapiovmsadapter.hpp +++ /dev/null @@ -1,66 +0,0 @@ -#pragma once -//***************************************************************************** -// Copyright 2023 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#include -#include -#include -#include -#include -#include -#include -#include - -#include // TODO fix path model_api/model_api/cpp/adapters/include/adapters/inference_adapter.h -#include - -// here we need to decide if we have several calculators (1 for OVMS repository, 1-N inside mediapipe) -// for the one inside OVMS repo it makes sense to reuse code from ovms lib - -class OVMS_Server_; -typedef struct OVMS_Server_ OVMS_Server; -namespace mediapipe { -namespace ovms { - -using InferenceOutput = std::map; -using InferenceInput = std::map; - -// TODO -// * why std::map -using shape_border_t = std::vector; -using shape_min_max_t = std::pair; -using shapes_min_max_t = std::unordered_map; -class OVMSInferenceAdapter : public ::InferenceAdapter { - OVMS_Server* cserver{nullptr}; - const std::string servableName; - uint32_t servableVersion; - std::vector inputNames; - std::vector outputNames; - shapes_min_max_t inShapesMinMaxes; - ov::AnyMap modelConfig; - -public: - OVMSInferenceAdapter(const std::string& servableName, uint32_t servableVersion = 0, OVMS_Server* server = nullptr); - virtual ~OVMSInferenceAdapter(); - InferenceOutput infer(const InferenceInput& input) override; - void loadModel(const std::shared_ptr& model, ov::Core& core, - const std::string& device, const ov::AnyMap& compilationConfig) override; - ov::PartialShape getInputShape(const std::string& inputName) const override; - std::vector getInputNames() const override; - std::vector getOutputNames() const override; - const ov::AnyMap& getModelConfig() const override; -}; -} // namespace ovms -} // namespace mediapipe diff --git a/mediapipe/calculators/ovms/openvinoinferencecalculator.cc b/mediapipe/calculators/ovms/openvinoinferencecalculator.cc deleted file mode 100644 index 6e52aa302a..0000000000 --- a/mediapipe/calculators/ovms/openvinoinferencecalculator.cc +++ /dev/null @@ -1,670 +0,0 @@ -//***************************************************************************** -// Copyright 2023 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#include -#include -#include -#include -#include -#include - -#include // TODO fix path model_api/model_api/cpp/adapters/include/adapters/inference_adapter.h -#include -#include - -#include "ovms.h" // NOLINT -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wdeprecated-declarations" -#include "tensorflow/core/framework/tensor.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/port/canonical_errors.h" -#include "mediapipe/framework/formats/tensor.h" -#include "mediapipe/calculators/ovms/openvinoinferencecalculator.pb.h" -#include "tensorflow/lite/c/common.h" -#pragma GCC diagnostic pop -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wall" -#include "tensorflow/lite/interpreter.h" -#pragma GCC diagnostic pop -// here we need to decide if we have several calculators (1 for OVMS repository, 1-N inside mediapipe) -// for the one inside OVMS repo it makes sense to reuse code from ovms lib -namespace mediapipe { -using std::endl; - -namespace { - -#define ASSERT_CIRCULAR_ERR(C_API_CALL) \ - { \ - auto* fatalErr = C_API_CALL; \ - RET_CHECK(fatalErr == nullptr); \ - } - -#define ASSERT_CAPI_STATUS_NULL(C_API_CALL) \ - { \ - auto* err = C_API_CALL; \ - if (err != nullptr) { \ - uint32_t code = 0; \ - const char* msg = nullptr; \ - ASSERT_CIRCULAR_ERR(OVMS_StatusCode(err, &code)); \ - ASSERT_CIRCULAR_ERR(OVMS_StatusDetails(err, &msg)); \ - LOG(INFO) << "Error encountred in OVMSCalculator:" << msg << " code: " << code; \ - OVMS_StatusDelete(err); \ - RET_CHECK(err == nullptr); \ - } \ - } -#define CREATE_GUARD(GUARD_NAME, CAPI_TYPE, CAPI_PTR) \ - std::unique_ptr GUARD_NAME(CAPI_PTR, &(CAPI_TYPE##Delete)); - -} // namespace - -const std::string SESSION_TAG{"SESSION"}; -const std::string OVTENSOR_TAG{"OVTENSOR"}; -const std::string OVTENSORS_TAG{"OVTENSORS"}; -const std::string TFTENSOR_TAG{"TFTENSOR"}; -const std::string TFTENSORS_TAG{"TFTENSORS"}; -const std::string MPTENSOR_TAG{"TENSOR"}; -const std::string MPTENSORS_TAG{"TENSORS"}; -const std::string TFLITE_TENSOR_TAG{"TFLITE_TENSOR"}; -const std::string TFLITE_TENSORS_TAG{"TFLITE_TENSORS"}; - -using TFSDataType = tensorflow::DataType; - -// Function from ovms/src/string_utils.h -bool startsWith(const std::string& str, const std::string& prefix) { - auto it = prefix.begin(); - bool sizeCheck = (str.size() >= prefix.size()); - if (!sizeCheck) { - return false; - } - bool allOf = std::all_of(str.begin(), - std::next(str.begin(), prefix.size()), - [&it](const char& c) { - return c == *(it++); - }); - return allOf; -} - -TFSDataType getPrecisionAsDataType(ov::element::Type_t precision) { - static std::unordered_map precisionMap{ - {ov::element::Type_t::f32, TFSDataType::DT_FLOAT}, - {ov::element::Type_t::f64, TFSDataType::DT_DOUBLE}, - {ov::element::Type_t::f16, TFSDataType::DT_HALF}, - {ov::element::Type_t::i64, TFSDataType::DT_INT64}, - {ov::element::Type_t::i32, TFSDataType::DT_INT32}, - {ov::element::Type_t::i16, TFSDataType::DT_INT16}, - {ov::element::Type_t::i8, TFSDataType::DT_INT8}, - {ov::element::Type_t::u64, TFSDataType::DT_UINT64}, - {ov::element::Type_t::u16, TFSDataType::DT_UINT16}, - {ov::element::Type_t::u8, TFSDataType::DT_UINT8}, - {ov::element::Type_t::boolean, TFSDataType::DT_BOOL} - }; - auto it = precisionMap.find(precision); - if (it == precisionMap.end()) { - return TFSDataType::DT_INVALID; - } - return it->second; -} - -static Tensor::ElementType OVType2MPType(ov::element::Type_t precision) { - static std::unordered_map precisionMap{ -// {ov::element::Type_t::f64, Tensor::ElementType::}, - {ov::element::Type_t::f32, Tensor::ElementType::kFloat32}, - {ov::element::Type_t::f16, Tensor::ElementType::kFloat16}, -// {ov::element::Type_t::i64, Tensor::ElementType::64}, - {ov::element::Type_t::i32, Tensor::ElementType::kInt32}, -// {ov::element::Type_t::i16, Tensor::ElementType::}, - {ov::element::Type_t::i8, Tensor::ElementType::kInt8}, -// {ov::element::Type_t::u64, Tensor::ElementType::64}, -// {ov::element::Type_t::u32, Tensor::ElementType::}, -// {ov::element::Type_t::u16, Tensor::ElementType::}, - {ov::element::Type_t::u8, Tensor::ElementType::kUInt8}, - {ov::element::Type_t::boolean, Tensor::ElementType::kBool} -// {ov::element::Type_t::, Tensor::ElementType::kChar} - }; - auto it = precisionMap.find(precision); - if (it == precisionMap.end()) { - return Tensor::ElementType::kNone; - } - return it->second; -} - -static ov::element::Type_t MPType2OVType(Tensor::ElementType precision) { - static std::unordered_map precisionMap{ -// Tensor::ElementType::, ov::element::Type_t::f64}, - {Tensor::ElementType::kFloat32, ov::element::Type_t::f32}, - {Tensor::ElementType::kFloat16, ov::element::Type_t::f16}, -// {Tensor::ElementType::, ov::element::Type_t::i64}, - {Tensor::ElementType::kInt32, ov::element::Type_t::i32}, -// {Tensor::ElementType::, ov::element::Type_t::i16}, - {Tensor::ElementType::kInt8, ov::element::Type_t::i8}, -// {Tensor::ElementType::, ov::element::Type_t::u64}, -// {Tensor::ElementType::, ov::element::Type_t::u32}, -// {Tensor::ElementType::, ov::element::Type_t::u16}, - {Tensor::ElementType::kUInt8, ov::element::Type_t::u8}, - {Tensor::ElementType::kBool, ov::element::Type_t::boolean} -// {Tensor::ElementType::kChar, ov::element::Type_t::} - }; - auto it = precisionMap.find(precision); - if (it == precisionMap.end()) { - return ov::element::Type_t::undefined; - } - return it->second; -} - -static ov::Tensor convertMPTensor2OVTensor(const Tensor& inputTensor) { - void* data; - switch(inputTensor.element_type()) { - case Tensor::ElementType::kFloat32: - case Tensor::ElementType::kFloat16: - data = reinterpret_cast(const_cast(inputTensor.GetCpuReadView().buffer())); - break; - case Tensor::ElementType::kUInt8: - data = reinterpret_cast(const_cast(inputTensor.GetCpuReadView().buffer())); - break; - case Tensor::ElementType::kInt8: - data = reinterpret_cast(const_cast(inputTensor.GetCpuReadView().buffer())); - break; - case Tensor::ElementType::kInt32: - data = reinterpret_cast(const_cast(inputTensor.GetCpuReadView().buffer())); - break; - case Tensor::ElementType::kBool: - data = reinterpret_cast(const_cast(inputTensor.GetCpuReadView().buffer())); - break; - default: - data = reinterpret_cast(const_cast(inputTensor.GetCpuReadView().buffer())); - break; - } - auto datatype = MPType2OVType(inputTensor.element_type());; - if (datatype == ov::element::Type_t::undefined) { - std::stringstream ss; - LOG(INFO) << "Not supported precision for Mediapipe tensor deserialization"; - std::runtime_error exc("Not supported precision for Mediapipe tensor deserialization"); - throw exc; - } - ov::Shape shape; - for (const auto& dim : inputTensor.shape().dims) { - shape.emplace_back(dim); - } - ov::Tensor result(datatype, shape, data); - return result; -} - -static Tensor convertOVTensor2MPTensor(const ov::Tensor& inputTensor) { - std::vector rawShape; - for (size_t i = 0; i < inputTensor.get_shape().size(); i++) { - rawShape.emplace_back(inputTensor.get_shape()[i]); - } - Tensor::Shape shape{rawShape}; - auto datatype = OVType2MPType(inputTensor.get_element_type()); - if (datatype == mediapipe::Tensor::ElementType::kNone) { - std::stringstream ss; - LOG(INFO) << "Not supported precision for Mediapipe tensor serialization: " << inputTensor.get_element_type(); - std::runtime_error exc("Not supported precision for Mediapipe tensor serialization"); - throw exc; - } - Tensor outputTensor(datatype, shape); - void* data; - switch(inputTensor.get_element_type()) { - case ov::element::Type_t::f32: - case ov::element::Type_t::f16: - data = reinterpret_cast(const_cast(outputTensor.GetCpuWriteView().buffer())); - break; - case ov::element::Type_t::u8: - data = reinterpret_cast(const_cast(outputTensor.GetCpuWriteView().buffer())); - break; - case ov::element::Type_t::i8: - data = reinterpret_cast(const_cast(outputTensor.GetCpuWriteView().buffer())); - break; - case ov::element::Type_t::i32: - data = reinterpret_cast(const_cast(outputTensor.GetCpuWriteView().buffer())); - break; - case ov::element::Type_t::boolean: - data = reinterpret_cast(const_cast(outputTensor.GetCpuWriteView().buffer())); - break; - default: - data = reinterpret_cast(const_cast(outputTensor.GetCpuWriteView().buffer())); - break; - } - std::memcpy(data, inputTensor.data(), inputTensor.get_byte_size()); - return outputTensor; -} - -ov::element::Type_t TFSPrecisionToIE2Precision(TFSDataType precision) { - static std::unordered_map precisionMap{ - {TFSDataType::DT_DOUBLE, ov::element::Type_t::f64}, - {TFSDataType::DT_FLOAT, ov::element::Type_t::f32}, - {TFSDataType::DT_HALF, ov::element::Type_t::f16}, - {TFSDataType::DT_INT64, ov::element::Type_t::i64}, - {TFSDataType::DT_INT32, ov::element::Type_t::i32}, - {TFSDataType::DT_INT16, ov::element::Type_t::i16}, - {TFSDataType::DT_INT8, ov::element::Type_t::i8}, - {TFSDataType::DT_UINT64, ov::element::Type_t::u64}, - {TFSDataType::DT_UINT32, ov::element::Type_t::u32}, - {TFSDataType::DT_UINT16, ov::element::Type_t::u16}, - {TFSDataType::DT_UINT8, ov::element::Type_t::u8}, - {TFSDataType::DT_BOOL, ov::element::Type_t::boolean}, - // {Precision::MIXED, ov::element::Type_t::MIXED}, - // {Precision::Q78, ov::element::Type_t::Q78}, - // {Precision::BIN, ov::element::Type_t::BIN}, - // {Precision::CUSTOM, ov::element::Type_t::CUSTOM - }; - auto it = precisionMap.find(precision); - if (it == precisionMap.end()) { - return ov::element::Type_t::undefined; - } - return it->second; -} - -static tensorflow::Tensor convertOVTensor2TFTensor(const ov::Tensor& t) { - using tensorflow::Tensor; - using tensorflow::TensorShape; - auto datatype = getPrecisionAsDataType(t.get_element_type()); - if (datatype == TFSDataType::DT_INVALID) { - std::stringstream ss; - LOG(INFO) << "Not supported precision for Tensorflow tensor serialization: " << t.get_element_type(); - std::runtime_error exc("Not supported precision for Tensorflow tensor serialization"); - throw exc; - } - TensorShape tensorShape; - std::vector rawShape; - for (size_t i = 0; i < t.get_shape().size(); i++) { - rawShape.emplace_back(t.get_shape()[i]); - } - int64_t dimsCount = rawShape.size(); - tensorflow::TensorShapeUtils::MakeShape(rawShape.data(), dimsCount, &tensorShape); - TensorShape::BuildTensorShapeBase(rawShape, static_cast*>(&tensorShape)); - // here we allocate default TF CPU allocator - tensorflow::Tensor result(datatype, tensorShape); - void* tftensordata = result.data(); - std::memcpy(tftensordata, t.data(), t.get_byte_size()); - return result; -} - -static ov::Tensor convertTFTensor2OVTensor(const tensorflow::Tensor& t) { - void* data = t.data(); - auto datatype = TFSPrecisionToIE2Precision(t.dtype()); - if (datatype == ov::element::Type_t::undefined) { - std::stringstream ss; - LOG(INFO) << "Not supported precision for Tensorflow tensor deserialization: " << t.dtype(); - std::runtime_error exc("Not supported precision for Tensorflow tensor deserialization"); - throw exc; - } - ov::Shape shape; - for (const auto& dim : t.shape()) { - shape.emplace_back(dim.size); - } - if (ov::shape_size(shape) <= 0) - return ov::Tensor(datatype, shape); // OV does not allow nullptr as data - return ov::Tensor(datatype, shape, data); -} - - -static ov::Tensor convertTFLiteTensor2OVTensor(const TfLiteTensor& t) { - void* data = t.data.f; // probably works only for floats - auto datatype = ov::element::f32; - ov::Shape shape; - // for some reason TfLite tensor does not have bs dim - shape.emplace_back(1); - // TODO: Support scalars and no data tensors with 0-dim - for (int i = 0; i < t.dims->size; ++i) { - shape.emplace_back(t.dims->data[i]); - } - ov::Tensor result(datatype, shape, data); - return result; -} - -class OpenVINOInferenceCalculator : public CalculatorBase { - std::shared_ptr<::InferenceAdapter> session{nullptr}; - std::unordered_map outputNameToTag; - std::vector input_order_list; - std::vector output_order_list; - std::unique_ptr interpreter_ = absl::make_unique(); - bool initialized = false; - -public: - static absl::Status GetContract(CalculatorContract* cc) { - LOG(INFO) << "OpenVINOInferenceCalculator GetContract start"; - RET_CHECK(!cc->Inputs().GetTags().empty()); - RET_CHECK(!cc->Outputs().GetTags().empty()); - RET_CHECK(cc->InputSidePackets().HasTag(SESSION_TAG)); - for (const std::string& tag : cc->Inputs().GetTags()) { - // could be replaced with absl::StartsWith when migrated to MP - if (startsWith(tag, OVTENSORS_TAG)) { - LOG(INFO) << "setting input tag:" << tag << " to OVTensors"; - cc->Inputs().Tag(tag).Set>(); - } else if (startsWith(tag, OVTENSOR_TAG)) { - LOG(INFO) << "setting input tag:" << tag << " to OVTensor"; - cc->Inputs().Tag(tag).Set(); - } else if (startsWith(tag, MPTENSORS_TAG)) { - LOG(INFO) << "setting input tag:" << tag << " to MPTensors"; - cc->Inputs().Tag(tag).Set>(); - } else if (startsWith(tag, MPTENSOR_TAG)) { - LOG(INFO) << "setting input tag:" << tag << " to MPTensor"; - cc->Inputs().Tag(tag).Set(); - } else if (startsWith(tag, TFTENSORS_TAG)) { - LOG(INFO) << "setting input tag:" << tag << " to TFTensors"; - cc->Inputs().Tag(tag).Set>(); - } else if (startsWith(tag, TFTENSOR_TAG)) { - LOG(INFO) << "setting input tag:" << tag << " to TFTensor"; - cc->Inputs().Tag(tag).Set(); - } else if (startsWith(tag, TFLITE_TENSORS_TAG)) { - LOG(INFO) << "setting input tag:" << tag << " to TFLITE_Tensors"; - cc->Inputs().Tag(tag).Set>(); - } else if (startsWith(tag, TFLITE_TENSOR_TAG)) { - LOG(INFO) << "setting input tag:" << tag << " to TFLITE_Tensor"; - cc->Inputs().Tag(tag).Set(); - } else { - LOG(INFO) << "setting input tag:" << tag << " to OVTensor"; - cc->Inputs().Tag(tag).Set(); - } - } - for (const std::string& tag : cc->Outputs().GetTags()) { - if (startsWith(tag, OVTENSORS_TAG)) { - LOG(INFO) << "setting input tag:" << tag << " to std::vector"; - cc->Outputs().Tag(tag).Set>(); - } else if (startsWith(tag, OVTENSOR_TAG)) { - LOG(INFO) << "setting input tag:" << tag << " to OVTensor"; - cc->Outputs().Tag(tag).Set(); - } else if (startsWith(tag, MPTENSORS_TAG)) { - LOG(INFO) << "setting input tag:" << tag << " to MPTensor"; - cc->Outputs().Tag(tag).Set>(); - } else if (startsWith(tag, MPTENSOR_TAG)) { - LOG(INFO) << "setting input tag:" << tag << " to MPTensor"; - cc->Outputs().Tag(tag).Set(); - } else if (startsWith(tag, TFTENSORS_TAG)) { - LOG(INFO) << "setting input tag:" << tag << " to TFTensor"; - cc->Outputs().Tag(tag).Set>(); - } else if (startsWith(tag, TFTENSOR_TAG)) { - LOG(INFO) << "setting input tag:" << tag << " to TFTensor"; - cc->Outputs().Tag(tag).Set(); - } else if (startsWith(tag, TFLITE_TENSORS_TAG)) { - LOG(INFO) << "setting input tag:" << tag << " to TFLITE_Tensor"; - cc->Outputs().Tag(tag).Set>(); - } else if (startsWith(tag, TFLITE_TENSOR_TAG)) { - LOG(INFO) << "setting input tag:" << tag << " to TFLITE_Tensor"; - cc->Outputs().Tag(tag).Set(); - } else { - LOG(INFO) << "setting output tag:" << tag << " to OVTensor"; - cc->Outputs().Tag(tag).Set(); - } - } - cc->InputSidePackets().Tag(SESSION_TAG.c_str()).Set>(); - LOG(INFO) << "OpenVINOInferenceCalculator GetContract end"; - return absl::OkStatus(); - } - - absl::Status Close(CalculatorContext* cc) final { - LOG(INFO) << "OpenVINOInferenceCalculator Close"; - return absl::OkStatus(); - } - absl::Status Open(CalculatorContext* cc) final { - LOG(INFO) << "OpenVINOInferenceCalculator Open start"; - session = cc->InputSidePackets() - .Tag(SESSION_TAG.c_str()) - .Get>(); - for (CollectionItemId id = cc->Inputs().BeginId(); - id < cc->Inputs().EndId(); ++id) { - if (!cc->Inputs().Get(id).Header().IsEmpty()) { - cc->Outputs().Get(id).SetHeader(cc->Inputs().Get(id).Header()); - } - } - if (cc->OutputSidePackets().NumEntries() != 0) { - for (CollectionItemId id = cc->InputSidePackets().BeginId(); - id < cc->InputSidePackets().EndId(); ++id) { - cc->OutputSidePackets().Get(id).Set(cc->InputSidePackets().Get(id)); - } - } - const auto& options = cc->Options(); - for (const auto& [key, value] : options.tag_to_output_tensor_names()) { - outputNameToTag[value] = key; - } - - auto& input_list = options.input_order_list(); - input_order_list.clear(); - for(int i = 0; i < input_list.size(); i++){ - input_order_list.push_back(input_list[i]); - } - auto& output_list = options.output_order_list(); - output_order_list.clear(); - for(int i = 0; i < output_list.size(); i++){ - output_order_list.push_back(output_list[i]); - } - - cc->SetOffset(TimestampDiff(0)); - LOG(INFO) << "OpenVINOInferenceCalculator Open end"; - return absl::OkStatus(); - } - - absl::Status Process(CalculatorContext* cc) final { - LOG(INFO) << "OpenVINOInferenceCalculator process start"; - if (cc->Inputs().NumEntries() == 0) { - return tool::StatusStop(); - } - ///////////////////// - // PREPARE INPUT MAP - ///////////////////// - - const auto& options = cc->Options(); - const auto& inputTagInputMap = options.tag_to_input_tensor_names(); - ::InferenceInput input; - ::InferenceOutput output; - for (const std::string& tag : cc->Inputs().GetTags()) { - const char* realInputName{nullptr}; - auto it = inputTagInputMap.find(tag); - if (it == inputTagInputMap.end()) { - realInputName = tag.c_str(); - } else { - realInputName = it->second.c_str(); - } -#define DESERIALIZE_TENSORS(TYPE, DESERIALIZE_FUN) \ - auto& packet = cc->Inputs().Tag(tag).Get>(); \ - if ( packet.size() > 1 && input_order_list.size() != packet.size()) { \ - LOG(INFO) << "input_order_list not set properly in options for multiple inputs."; \ - RET_CHECK(false); \ - } \ - if (this->input_order_list.size() > 0){ \ - for (size_t i = 0; i < this->input_order_list.size(); i++) { \ - auto& tensor = packet[i]; \ - input[this->input_order_list[i]] = DESERIALIZE_FUN(tensor); \ - } \ - } else if (packet.size() == 1) { \ - input[realInputName] = DESERIALIZE_FUN(packet[0]); \ - } - try { - if (startsWith(tag, OVTENSORS_TAG)) { - DESERIALIZE_TENSORS(ov::Tensor,); - } else if (startsWith(tag, TFLITE_TENSORS_TAG)) { - DESERIALIZE_TENSORS(TfLiteTensor, convertTFLiteTensor2OVTensor); - } else if (startsWith(tag, MPTENSORS_TAG)) { - DESERIALIZE_TENSORS(Tensor, convertMPTensor2OVTensor); - } else if (startsWith(tag, OVTENSOR_TAG)) { - auto& packet = cc->Inputs().Tag(tag).Get(); - input[realInputName] = packet; - } else if (startsWith(tag, TFLITE_TENSOR_TAG)) { - auto& packet = cc->Inputs().Tag(tag).Get(); - input[realInputName] = convertTFLiteTensor2OVTensor(packet); - } else if (startsWith(tag, MPTENSOR_TAG)) { - auto& packet = cc->Inputs().Tag(tag).Get(); - input[realInputName] = convertMPTensor2OVTensor(packet); - } else if (startsWith(tag, TFTENSOR_TAG)) { - auto& packet = cc->Inputs().Tag(tag).Get(); - input[realInputName] = convertTFTensor2OVTensor(packet); - } else { - auto& packet = cc->Inputs().Tag(tag).Get(); - input[realInputName] = packet; - } - } catch (const std::runtime_error& e) { - LOG(INFO) << "Failed to deserialize tensor error:" << e.what(); - RET_CHECK(false); - } - } - ////////////////// - // INFERENCE - ////////////////// - try { - output = session->infer(input); - } catch (const std::exception& e) { - LOG(INFO) << "Caught exception from session infer():" << e.what(); - RET_CHECK(false); - } catch (...) { - LOG(INFO) << "Caught unknown exception from session infer()"; - RET_CHECK(false); - } - auto outputsCount = output.size(); - RET_CHECK(outputsCount >= cc->Outputs().GetTags().size()); - LOG(INFO) << "output tags size: " << cc->Outputs().GetTags().size(); - for (const auto& tag : cc->Outputs().GetTags()) { - LOG(INFO) << "Processing tag: " << tag; - std::string tensorName; - auto it = options.tag_to_output_tensor_names().find(tag); - if (it == options.tag_to_output_tensor_names().end()) { - tensorName = tag; - } else { - tensorName = it->second; - } - auto tensorIt = output.find(tensorName); - if (tensorIt == output.end()) { - LOG(INFO) << "Could not find: " << tensorName << " in inference output"; - RET_CHECK(false); - } - try { - if (startsWith(tag, OVTENSORS_TAG)) { - LOG(INFO) << "OVMS calculator will process vector"; - auto tensors = std::make_unique>(); - if ( output.size() > 1 && this->output_order_list.size() != this->output_order_list.size()) - { - LOG(INFO) << "output_order_list not set properly in options for multiple outputs."; - RET_CHECK(false); - } - if (this->output_order_list.size() > 0) { - for (size_t i = 0; i < this->output_order_list.size(); i++) { - tensorName = this->output_order_list[i]; - tensorIt = output.find(tensorName); - if (tensorIt == output.end()) { - LOG(INFO) << "Could not find: " << tensorName << " in inference output"; - RET_CHECK(false); - } - tensors->emplace_back(tensorIt->second); - } - } else { - for (auto& [name,tensor] : output) { - tensors->emplace_back(tensor); - } - } - cc->Outputs().Tag(tag).Add( - tensors.release(), - cc->InputTimestamp()); - //break; // TODO FIXME order of outputs - // no need to break since we only have one tag - // create concatenator calc - } else if (startsWith(tag, MPTENSORS_TAG)) { - LOG(INFO) << "OVMS calculator will process vector"; - auto tensors = std::make_unique>(); - if ( output.size() > 1 && this->output_order_list.size() != this->output_order_list.size()) - { - LOG(INFO) << "output_order_list not set properly in options for multiple outputs."; - RET_CHECK(false); - } - if (this->output_order_list.size() > 0) { - for (size_t i = 0; i < this->output_order_list.size(); i++) { - tensorName = this->output_order_list[i]; - tensorIt = output.find(tensorName); - if (tensorIt == output.end()) { - LOG(INFO) << "Could not find: " << tensorName << " in inference output"; - RET_CHECK(false); - } - tensors->emplace_back(convertOVTensor2MPTensor(tensorIt->second)); - } - } else { - for (auto& [name,tensor] : output) { - tensors->emplace_back(convertOVTensor2MPTensor(tensor)); - } - } - cc->Outputs().Tag(tag).Add( - tensors.release(), - cc->InputTimestamp()); - //break; // TODO FIXME order of outputs - // no need to break since we only have one tag - // create concatenator calc - } else if (startsWith(tag, TFLITE_TENSORS_TAG)) { - // TODO FIXME use output_order_list - LOG(INFO) << "OVMS calculator will process vector"; - auto outputStreamTensors = std::vector(); - if (!this->initialized) { - interpreter_->AddTensors(output.size()); - std::vector indexes(output.size()); - std::iota(indexes.begin(), indexes.end(), 0); - interpreter_->SetInputs(indexes); - size_t tensorId = 0; - for (auto& [name,tensor] : output) { - std::vector tfliteshape; - for (auto& d : tensor.get_shape()) { - tfliteshape.emplace_back(d); - } - interpreter_->SetTensorParametersReadWrite( - tensorId, - kTfLiteFloat32, // TODO datatype - name.c_str(), - tfliteshape, - TfLiteQuantization()); - ++tensorId; - } - interpreter_->AllocateTensors(); - this->initialized = true; - } - size_t tensorId = 0; - for (auto& [name,tensor] : output) { - const int interpreterTensorId = interpreter_->inputs()[tensorId]; - TfLiteTensor* tflitetensor = interpreter_->tensor(interpreterTensorId); - void* tensor_ptr = tflitetensor->data.f; - std::memcpy(tensor_ptr, tensor.data(), tensor.get_byte_size()); - outputStreamTensors.emplace_back(*tflitetensor); - ++tensorId; - } - cc->Outputs().Tag(tag).AddPacket(MakePacket>(std::move(outputStreamTensors)).At( cc->InputTimestamp())); - break; - } else if (startsWith(tag, OVTENSOR_TAG)) { - LOG(INFO) << "OVMS calculator will process ov::Tensor"; - cc->Outputs().Tag(tag).Add( - new ov::Tensor(tensorIt->second), - cc->InputTimestamp()); - } else if (startsWith(tag, TFTENSOR_TAG)) { - LOG(INFO) << "OVMS calculator will process tensorflow::Tensor"; - cc->Outputs().Tag(tag).Add( - new tensorflow::Tensor(convertOVTensor2TFTensor(tensorIt->second)), - cc->InputTimestamp()); - } else if (startsWith(tag, MPTENSOR_TAG)) { - LOG(INFO) << "OVMS calculator will process mediapipe::Tensor"; - cc->Outputs().Tag(tag).Add( - new Tensor(convertOVTensor2MPTensor(tensorIt->second)), - cc->InputTimestamp()); - } else { - LOG(INFO) << "OVMS calculator will process ov::Tensor"; - cc->Outputs().Tag(tag).Add( - new ov::Tensor(tensorIt->second), - cc->InputTimestamp()); - } - } catch (const std::runtime_error& e) { - LOG(INFO) << "Failed to deserialize tensor error:" << e.what(); - RET_CHECK(false); - } - } - LOG(INFO) << "OpenVINOInferenceCalculator process end"; - return absl::OkStatus(); - } -}; - -REGISTER_CALCULATOR(OpenVINOInferenceCalculator); -} // namespace mediapipe diff --git a/mediapipe/calculators/ovms/openvinoinferencecalculator.h b/mediapipe/calculators/ovms/openvinoinferencecalculator.h deleted file mode 100644 index 8a92b8120c..0000000000 --- a/mediapipe/calculators/ovms/openvinoinferencecalculator.h +++ /dev/null @@ -1,44 +0,0 @@ -#pragma once -//***************************************************************************** -// Copyright 2023 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#include -#include - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wdeprecated-declarations" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/port/canonical_errors.h" -#pragma GCC diagnostic pop -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wall" -#include "tensorflow/lite/interpreter.h" -#pragma GCC diagnostic pop -class InferenceAdapter; -namespace mediapipe { -class OpenVINOInferenceCalculator : public CalculatorBase { - std::shared_ptr<::InferenceAdapter> session{nullptr}; - std::unordered_map outputNameToTag; - std::vector input_order_list; - std::vector output_order_list; - std::unique_ptr interpreter_ = absl::make_unique(); - bool initialized = false; -public: - static absl::Status GetContract(CalculatorContract* cc); - absl::Status Close(CalculatorContext* cc) override final; - absl::Status Open(CalculatorContext* cc) override final; - absl::Status Process(CalculatorContext* cc) override final; -}; -} // namespace mediapipe diff --git a/mediapipe/calculators/ovms/openvinoinferencecalculator.proto b/mediapipe/calculators/ovms/openvinoinferencecalculator.proto deleted file mode 100644 index e5b36112a6..0000000000 --- a/mediapipe/calculators/ovms/openvinoinferencecalculator.proto +++ /dev/null @@ -1,33 +0,0 @@ -//***************************************************************************** -// Copyright 2023 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** - -syntax = "proto2"; -package mediapipe; - -import "mediapipe/framework/calculator.proto"; - -message OpenVINOInferenceCalculatorOptions { - extend mediapipe.CalculatorOptions { - // https://github.com/google/mediapipe/issues/634 have to be unique in app - // no rule to obtain this - optional OpenVINOInferenceCalculatorOptions ext = 113473743; - } - map tag_to_input_tensor_names = 1; - map tag_to_output_tensor_names = 2; - // repeated gives optional option by default - repeated string input_order_list = 3; - repeated string output_order_list = 4; -} diff --git a/mediapipe/calculators/ovms/openvinoinferencecalculator_test.cc b/mediapipe/calculators/ovms/openvinoinferencecalculator_test.cc deleted file mode 100644 index 6e0d857ccf..0000000000 --- a/mediapipe/calculators/ovms/openvinoinferencecalculator_test.cc +++ /dev/null @@ -1,179 +0,0 @@ -//***************************************************************************** -// Copyright 2023 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#include "openvinoinferencecalculator.h" -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -#include "ovms.h" // NOLINT -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wdeprecated-declarations" -#include "mediapipe/calculators/ovms/openvinoinferencecalculator.pb.h" -#include "mediapipe/framework/port/canonical_errors.h" -#include "mediapipe/framework/deps/status_matchers.h" -#include "mediapipe/framework/formats/tensor.h" -#include "mediapipe/framework/port/canonical_errors.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/parse_text_proto.h" -#include "tensorflow/core/framework/tensor.h" -#pragma GCC diagnostic pop -using mediapipe::Adopt; -using mediapipe::CalculatorContract; -using mediapipe::CalculatorGraph; -using mediapipe::CalculatorGraphConfig; -using mediapipe::OpenVINOInferenceCalculator; -using mediapipe::CalculatorState; -using mediapipe::ParseTextProtoOrDie; -using mediapipe::Packet; -using mediapipe::PacketType; -using mediapipe::Timestamp; - -class OpenVINOInferenceCalculatorTest : public ::testing::Test { -PacketType OVTENSOR_TYPE; -PacketType OVTENSORS_TYPE; -PacketType MPTENSOR_TYPE; -PacketType MPTENSORS_TYPE; -public: - void SetUp() override { - OVTENSOR_TYPE.Set(); - OVTENSORS_TYPE.Set>(); - MPTENSOR_TYPE.Set(); - MPTENSORS_TYPE.Set>(); - } -}; - -TEST_F(OpenVINOInferenceCalculatorTest, VerifySupportedTags) { - auto calculator = - mediapipe::ParseTextProtoOrDie( - R"pb( - calculator: "OpenVINOInferenceCalculator" - input_side_packet: "SESSION:not_used_session" - input_stream: "OVTENSOR:input" - input_stream: "MPTENSOR:input" - input_stream: "TFTENSOR:input" - input_stream: "TFLITE_TENSOR:input" - output_stream: "OVTENSORS:output" - output_stream: "MPTENSORS:output" - output_stream: "TFTENSORS:output" - output_stream: "TFLITE_TENSORS:output" - )pb"); - auto cc = absl::make_unique(); - cc->Initialize(calculator); - auto abslStatus = mediapipe::OpenVINOInferenceCalculator::GetContract(cc.get()); - ASSERT_EQ(abslStatus.code(), absl::StatusCode::kOk) << abslStatus.message(); - EXPECT_EQ(1, cc->InputSidePackets().TagMap()->NumEntries()); - EXPECT_EQ(0, cc->OutputSidePackets().NumEntries()); - auto& inputPacketsTags = cc->Inputs(); - auto& outputPacketsTags = cc->Outputs(); - EXPECT_EQ(4, inputPacketsTags.TagMap()->NumEntries()); - EXPECT_EQ(4, outputPacketsTags.TagMap()->NumEntries()); -} -TEST_F(OpenVINOInferenceCalculatorTest, VerifyNotAllowedEmptySideInputPacket) { - auto calculator = - mediapipe::ParseTextProtoOrDie( - R"pb( - calculator: "OpenVINOInferenceCalculator" - input_stream: "OVTENSOR:input" - output_stream: "OVTENSOR:output" - )pb"); - auto cc = absl::make_unique(); - cc->Initialize(calculator); - auto abslStatus = mediapipe::OpenVINOInferenceCalculator::GetContract(cc.get()); - EXPECT_EQ(abslStatus.code(), absl::StatusCode::kInternal) << abslStatus.message(); -} -TEST_F(OpenVINOInferenceCalculatorTest, VerifyNotAllowedSideOutputPacket) { - auto calculator = - mediapipe::ParseTextProtoOrDie( - R"pb( - calculator: "OpenVINOInferenceCalculator" - output_side_packet: "SESSION:not_used_session" - input_stream: "OVTENSOR:input" - output_stream: "OVTENSOR:output" - )pb"); - auto cc = absl::make_unique(); - cc->Initialize(calculator); - auto abslStatus = mediapipe::OpenVINOInferenceCalculator::GetContract(cc.get()); - EXPECT_EQ(abslStatus.code(), absl::StatusCode::kInternal) << abslStatus.message(); -} -TEST_F(OpenVINOInferenceCalculatorTest, BasicDummyInference) { - std::string graph_proto = R"( - input_stream: "input" - output_stream: "output" - node { - calculator: "OpenVINOModelServerSessionCalculator" - output_side_packet: "SESSION:session" - node_options: { - [type.googleapis.com / mediapipe.OpenVINOModelServerSessionCalculatorOptions]: { - servable_name: "dummy" - server_config: "/mediapipe/mediapipe/calculators/ovms/test_data/config.json" - } - } - } - node { - calculator: "OpenVINOInferenceCalculator" - input_side_packet: "SESSION:session" - input_stream: "OVTENSOR:input" - output_stream: "OVTENSOR:output" - node_options: { - [type.googleapis.com / mediapipe.OpenVINOInferenceCalculatorOptions]: { - output_order_list: ["Identity:0", "Identity_1:0"] - tag_to_input_tensor_names { - key: "OVTENSOR" - value: "b" - } - tag_to_output_tensor_names { - key: "OVTENSOR" - value: "a" - } - } - } - } - )"; - CalculatorGraphConfig graph_config = - ParseTextProtoOrDie(graph_proto); - const std::string inputStreamName = "input"; - const std::string outputStreamName = "output"; - // avoid creating pollers, retreiving packets etc. - std::vector output_packets; - mediapipe::tool::AddVectorSink(outputStreamName, &graph_config, &output_packets); - CalculatorGraph graph(graph_config); - MP_ASSERT_OK(graph.StartRun({})); - auto datatype = ov::element::Type_t::f32; - ov::Shape shape{1,10}; - std::vector data{0,1,2,3,4,5,6,7,8,9}; - auto inputTensor = std::make_unique(datatype, shape, data.data()); - MP_ASSERT_OK(graph.AddPacketToInputStream( - inputStreamName, Adopt(inputTensor.release()).At(Timestamp(0)))); - MP_ASSERT_OK(graph.CloseInputStream(inputStreamName)); - MP_ASSERT_OK(graph.WaitUntilIdle()); - ASSERT_EQ(1, output_packets.size()); - const ov::Tensor& outputTensor = - output_packets[0].Get(); - MP_ASSERT_OK(graph.WaitUntilDone()); - EXPECT_EQ(datatype, outputTensor.get_element_type()); - EXPECT_THAT(outputTensor.get_shape(), testing::ElementsAre(1,10)); - const void* outputData = outputTensor.data(); - for (size_t i = 0; i < data.size(); ++i) { - EXPECT_EQ(data[i] + 1, *(reinterpret_cast(outputData) + i)) << i; - } -} diff --git a/mediapipe/calculators/ovms/openvinomodelserversessioncalculator.cc b/mediapipe/calculators/ovms/openvinomodelserversessioncalculator.cc deleted file mode 100644 index 004025915b..0000000000 --- a/mediapipe/calculators/ovms/openvinomodelserversessioncalculator.cc +++ /dev/null @@ -1,210 +0,0 @@ -//***************************************************************************** -// Copyright 2023 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#include "openvinomodelserversessioncalculator.h" -#include -#include -#include -#include -#include -#include -#include - -#include - -#include "ovms.h" // NOLINT -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wdeprecated-declarations" -#include "mediapipe/calculators/ovms/openvinomodelserversessioncalculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/port/canonical_errors.h" -#include "modelapiovmsadapter.hpp" -#pragma GCC diagnostic pop -namespace mediapipe { -using ovms::OVMSInferenceAdapter; - -const std::string SESSION_TAG{"SESSION"}; -ov::Core UNUSED_OV_CORE; - -#define ASSERT_CIRCULAR_ERR(C_API_CALL) \ - { \ - auto* fatalErr = C_API_CALL; \ - RET_CHECK(fatalErr == nullptr); \ - } - -#define ASSERT_CAPI_STATUS_NULL(C_API_CALL) \ - { \ - auto* err = C_API_CALL; \ - if (err != nullptr) { \ - uint32_t code = 0; \ - const char* msg = nullptr; \ - ASSERT_CIRCULAR_ERR(OVMS_StatusCode(err, &code)); \ - ASSERT_CIRCULAR_ERR(OVMS_StatusDetails(err, &msg)); \ - LOG(INFO) << "Error encountred in OVMSCalculator:" << msg << " code: " << code; \ - OVMS_StatusDelete(err); \ - RET_CHECK(nullptr == err); \ - } \ - } - -#define REPORT_CAPI_STATUS_NULL(C_API_CALL) \ - { \ - auto* err = C_API_CALL; \ - if (err != nullptr) { \ - uint32_t code = 0; \ - const char* msg = nullptr; \ - ASSERT_CIRCULAR_ERR(OVMS_StatusCode(err, &code)); \ - ASSERT_CIRCULAR_ERR(OVMS_StatusDetails(err, &msg)); \ - LOG(INFO) << "Error encountred in OVMSCalculator:" << msg << " code: " << code; \ - OVMS_StatusDelete(err); \ - } \ - } - -// Function from ovms/src/string_utils.h -void erase_spaces(std::string& str) { - str.erase(std::remove_if(str.begin(), str.end(), - [](char c) -> bool { - return std::isspace(c, std::locale::classic()); - }), - str.end()); -} - -// Function from ovms/src/string_utils.h -std::optional stou32(const std::string& input) { - std::string str = input; - erase_spaces(str); - - if (str.size() > 0 && str[0] == '-') { - return std::nullopt; - } - - try { - uint64_t val = std::stoul(str); - if (val > std::numeric_limits::max()) { - return std::nullopt; - } - return {static_cast(val)}; - } catch (...) { - return std::nullopt; - } -} - -class SettingsGuard { -public: - OVMS_ServerSettings* serverSettings{nullptr}; - OVMS_ModelsSettings* modelsSettings{nullptr}; - SettingsGuard() { - OVMS_ServerSettingsNew(&serverSettings); - OVMS_ModelsSettingsNew(&modelsSettings); - } - ~SettingsGuard() { - OVMS_ServerSettingsDelete(serverSettings); - OVMS_ModelsSettingsDelete(modelsSettings); - } -}; - -absl::Status OpenVINOModelServerSessionCalculator::GetContract(CalculatorContract* cc) { - LOG(INFO) << "OpenVINOModelServerSessionCalculator GetContract start"; - RET_CHECK(cc->Inputs().GetTags().empty()); - RET_CHECK(cc->Outputs().GetTags().empty()); - cc->OutputSidePackets().Tag(SESSION_TAG.c_str()).Set>(); - const auto& options = cc->Options(); - RET_CHECK(!options.servable_name().empty()); - LOG(INFO) << "OpenVINOModelServerSessionCalculator GetContract end"; - return absl::OkStatus(); -} - -absl::Status OpenVINOModelServerSessionCalculator::Close(CalculatorContext* cc) { - LOG(INFO) << "OpenVINOModelServerSessionCalculator Close"; - return absl::OkStatus(); -} -absl::Status OpenVINOModelServerSessionCalculator::Open(CalculatorContext* cc) { - LOG(INFO) << "OpenVINOModelServerSessionCalculator Open start"; - for (CollectionItemId id = cc->Inputs().BeginId(); - id < cc->Inputs().EndId(); ++id) { - if (!cc->Inputs().Get(id).Header().IsEmpty()) { - cc->Outputs().Get(id).SetHeader(cc->Inputs().Get(id).Header()); - } - } - if (cc->OutputSidePackets().NumEntries() != 0) { - for (CollectionItemId id = cc->InputSidePackets().BeginId(); - id < cc->InputSidePackets().EndId(); ++id) { - cc->OutputSidePackets().Get(id).Set(cc->InputSidePackets().Get(id)); - } - } - cc->SetOffset(TimestampDiff(0)); - - const auto& options = cc->Options(); - // if config is in calc then we start the server - LOG(INFO) << "Will check if we want to start server"; - if (!options.server_config().empty()) { - // Lock access to server from multiple calculator instances during the model loading phase - std::unique_lock lk(OpenVINOModelServerSessionCalculator::loadingMtx); - bool isServerReady = false; - bool isServerLive = false; - OVMS_ServerNew(&cserver); - - ASSERT_CAPI_STATUS_NULL(OVMS_ServerLive(cserver, &isServerLive)); - - if (triedToStartOVMS) { - RET_CHECK(isServerLive); - } else if (!isServerLive) { - LOG(INFO) << "Will start new server"; - triedToStartOVMS = true; - SettingsGuard guard; - OVMS_ServerSettingsNew(&guard.serverSettings); - OVMS_ModelsSettingsNew(&guard.modelsSettings); - OVMS_ModelsSettingsSetConfigPath(guard.modelsSettings, options.server_config().c_str()); - LOG(INFO) << "state config file:" << options.server_config(); - OVMS_ServerSettingsSetLogLevel(guard.serverSettings, OVMS_LOG_DEBUG); - - ASSERT_CAPI_STATUS_NULL(OVMS_ServerStartFromConfigurationFile(cserver, guard.serverSettings, guard.modelsSettings)); - - ASSERT_CAPI_STATUS_NULL(OVMS_ServerReady(cserver, &isServerReady)); - RET_CHECK(isServerReady); - LOG(INFO) << "Server started"; - } - } - const std::string& servableName = options.servable_name(); - const std::string& servableVersionStr = options.servable_version(); - auto servableVersionOpt = stou32(servableVersionStr); - // 0 means default - uint32_t servableVersion = servableVersionOpt.value_or(0); - auto session = std::make_shared(servableName, servableVersion); - try { - session->loadModel(nullptr, UNUSED_OV_CORE, "UNUSED", {}); - } catch (const std::exception& e) { - LOG(INFO) << "Caught exception with message: " << e.what(); - RET_CHECK(false); - } catch (...) { - LOG(INFO) << "Caught unknown exception"; - RET_CHECK(false); - } - - LOG(INFO) << "OpenVINOModelServerSessionCalculator create adapter"; - cc->OutputSidePackets().Tag(SESSION_TAG.c_str()).Set(MakePacket>(session)); - LOG(INFO) << "OpenVINOModelServerSessionCalculator Open end"; - return absl::OkStatus(); -} - -absl::Status OpenVINOModelServerSessionCalculator::Process(CalculatorContext* cc) { - LOG(INFO) << "OpenVINOModelServerSessionCalculator Process"; - return absl::OkStatus(); -} - -bool OpenVINOModelServerSessionCalculator::triedToStartOVMS = false; -std::mutex OpenVINOModelServerSessionCalculator::loadingMtx; - -REGISTER_CALCULATOR(OpenVINOModelServerSessionCalculator); -} // namespace mediapipe diff --git a/mediapipe/calculators/ovms/openvinomodelserversessioncalculator.h b/mediapipe/calculators/ovms/openvinomodelserversessioncalculator.h deleted file mode 100644 index e21a58eb23..0000000000 --- a/mediapipe/calculators/ovms/openvinomodelserversessioncalculator.h +++ /dev/null @@ -1,51 +0,0 @@ -//***************************************************************************** -// Copyright 2023 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#include -#include -#include -#include -#include -#include -#include - -#include - -#include "ovms.h" // NOLINT -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wdeprecated-declarations" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/port/canonical_errors.h" -#include "modelapiovmsadapter.hpp" -#include "mediapipe/calculators/ovms/openvinomodelserversessioncalculator.pb.h" -#pragma GCC diagnostic pop -namespace mediapipe { - -using ovms::OVMSInferenceAdapter; - -class OpenVINOModelServerSessionCalculator : public CalculatorBase { - std::shared_ptr<::InferenceAdapter> adapter; - std::unordered_map outputNameToTag; - OVMS_Server* cserver{nullptr}; - static bool triedToStartOVMS; - static std::mutex loadingMtx; -public: - static absl::Status GetContract(CalculatorContract* cc); - absl::Status Close(CalculatorContext* cc) override final; - absl::Status Open(CalculatorContext* cc) override final; - - absl::Status Process(CalculatorContext* cc) override final; -}; -} // namespace mediapipe diff --git a/mediapipe/calculators/ovms/openvinomodelserversessioncalculator.proto b/mediapipe/calculators/ovms/openvinomodelserversessioncalculator.proto deleted file mode 100644 index eb63b15df0..0000000000 --- a/mediapipe/calculators/ovms/openvinomodelserversessioncalculator.proto +++ /dev/null @@ -1,33 +0,0 @@ -//***************************************************************************** -// Copyright 2023 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** - -syntax = "proto2"; -package mediapipe; - -import "mediapipe/framework/calculator.proto"; - -message OpenVINOModelServerSessionCalculatorOptions { - extend mediapipe.CalculatorOptions { - // https://github.com/google/mediapipe/issues/634 have to be unique in app - // no rule to obtain this - optional OpenVINOModelServerSessionCalculatorOptions ext = 113473744; - } - required string servable_name = 1; - optional string servable_version = 2; - // service_url: "13.21.212.171:9718" - optional string service_url = 3; - optional string server_config = 4; -} diff --git a/mediapipe/calculators/ovms/openvinomodelserversessioncalculator_test.cc b/mediapipe/calculators/ovms/openvinomodelserversessioncalculator_test.cc deleted file mode 100644 index 38955124e2..0000000000 --- a/mediapipe/calculators/ovms/openvinomodelserversessioncalculator_test.cc +++ /dev/null @@ -1,153 +0,0 @@ -//***************************************************************************** -// Copyright 2023 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#include "openvinomodelserversessioncalculator.h" -#include -#include -#include -#include -#include -#include - -#include "mediapipe/framework/port/gtest.h" - -#include // TODO fix path model_api/model_api/cpp/adapters/include/adapters/inference_adapter.h -#include -#include - -#include "ovms.h" // NOLINT -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wdeprecated-declarations" -#include "mediapipe/calculators/ovms/openvinomodelserversessioncalculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/port/canonical_errors.h" -#include "mediapipe/framework/deps/status_matchers.h" -//#include "mediapipe/framework/calculator_runner.h" -#include "mediapipe/framework/formats/tensor.h" -#include "mediapipe/framework/port/canonical_errors.h" -#include "mediapipe/framework/port/parse_text_proto.h" -#include "mediapipe/framework/tool/tag_map_helper.h" -#pragma GCC diagnostic pop -// here we need to decide if we have several calculators (1 for OVMS repository, 1-N inside mediapipe) -// for the one inside OVMS repo it makes sense to reuse code from ovms lib -using mediapipe::CalculatorContract; -using mediapipe::OpenVINOModelServerSessionCalculator; -using mediapipe::CalculatorState; -using mediapipe::ParseTextProtoOrDie; -using mediapipe::PacketType; - -class OpenVINOModelServerSessionCalculatorTest : public ::testing::Test { -PacketType OVTENSOR_TYPE; -PacketType OVTENSORS_TYPE; -PacketType MPTENSOR_TYPE; -PacketType MPTENSORS_TYPE; -public: - void SetUp() override { - OVTENSOR_TYPE.Set(); - OVTENSORS_TYPE.Set>(); - MPTENSOR_TYPE.Set(); - MPTENSORS_TYPE.Set>(); - } -}; - -TEST_F(OpenVINOModelServerSessionCalculatorTest, VerifyCorrectPbtxtWithAllOptions) { - auto node = - mediapipe::ParseTextProtoOrDie( - R"pb( - calculator: "OpenVINOModelServerSessionCalculator" - output_side_packet: "SESSION:session" - node_options: { - [type.googleapis.com / mediapipe.OpenVINOModelServerSessionCalculatorOptions]: { - servable_name: "not_used_name" - servable_version: "1" - server_config: "mediapipe/config.json" - service_url: "192.168.0.1:9178" - } - } - )pb"); - auto cc = absl::make_unique(); - cc->Initialize(node); - auto abslStatus = mediapipe::OpenVINOModelServerSessionCalculator::GetContract(cc.get()); - ASSERT_EQ(abslStatus.code(), absl::StatusCode::kOk) << abslStatus.message(); - EXPECT_EQ(0, cc->InputSidePackets().TagMap()->NumEntries()); - EXPECT_EQ(1, cc->OutputSidePackets().NumEntries()); - auto& inputPacketsTags = cc->Inputs(); - auto& outputPacketsTags = cc->Outputs(); - EXPECT_EQ(0, inputPacketsTags.TagMap()->NumEntries()); - EXPECT_EQ(0, outputPacketsTags.TagMap()->NumEntries()); -} -TEST_F(OpenVINOModelServerSessionCalculatorTest, VerifyOptionalityOfOptionFields) { - // servable_version, server_config, server_url are optional - auto node = - mediapipe::ParseTextProtoOrDie( - R"pb( - calculator: "OpenVINOModelServerSessionCalculator" - output_side_packet: "SESSION:session" - node_options: { - [type.googleapis.com / mediapipe.OpenVINOModelServerSessionCalculatorOptions]: { - servable_name: "not_used_name" - } - } - )pb"); - auto cc = absl::make_unique(); - cc->Initialize(node); - auto abslStatus = mediapipe::OpenVINOModelServerSessionCalculator::GetContract(cc.get()); - EXPECT_EQ(abslStatus.code(), absl::StatusCode::kOk) << abslStatus.message(); -} -TEST_F(OpenVINOModelServerSessionCalculatorTest, VerifyMandatorityOfFields) { - // servable_version, server_config, server_url are optional - mediapipe::CalculatorGraphConfig::Node node; - bool success = mediapipe::ParseTextProto( - R"pb( - calculator: "OpenVINOModelServerSessionCalculator" - output_side_packet: "SESSION:session" - node_options: { - [type.googleapis.com / mediapipe.OpenVINOModelServerSessionCalculatorOptions]: { - # commented out servable_name: "not_used_name" - servable_version: "1" - server_config: "mediapipe/config.json" - service_url: "192.168.0.1:9178" - } - } - )pb", &node); - EXPECT_FALSE(success); -} -TEST_F(OpenVINOModelServerSessionCalculatorTest, VerifyNonExistingFields) { - mediapipe::CalculatorGraphConfig::Node node; - bool success = mediapipe::ParseTextProto( - R"pb( - calculator: "OpenVINOModelServerSessionCalculator" - output_side_packet: "SESSION:session" - node_options: { - [type.googleapis.com / mediapipe.OpenVINOModelServerSessionCalculatorOptions]: { - servable_name: "not_used_name" - some_random_name: 1 - } - } - )pb", &node); - EXPECT_FALSE(success); -} -TEST_F(OpenVINOModelServerSessionCalculatorTest, MissingAllOptions) { - auto node = - mediapipe::ParseTextProtoOrDie( - R"pb( - calculator: "OpenVINOModelServerSessionCalculator" - output_side_packet: "SESSION:session" - )pb"); - auto cc = absl::make_unique(); - cc->Initialize(node); - auto abslStatus = mediapipe::OpenVINOModelServerSessionCalculator::GetContract(cc.get()); - ASSERT_EQ(abslStatus.code(), absl::StatusCode::kInternal) << abslStatus.message(); -} diff --git a/mediapipe/calculators/ovms/ovms_calculator.cc b/mediapipe/calculators/ovms/ovms_calculator.cc deleted file mode 100644 index 94249169ac..0000000000 --- a/mediapipe/calculators/ovms/ovms_calculator.cc +++ /dev/null @@ -1,281 +0,0 @@ -//***************************************************************************** -// Copyright 2023 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#include -#include -#include - -#include - -#include "../ovms.h" // NOLINT -#include "../stringutils.hpp" // TODO dispose - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wdeprecated-declarations" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/port/canonical_errors.h" -#pragma GCC diagnostic pop -#include "src/mediapipe_calculators/ovmscalculator.pb.h" -// here we need to decide if we have several calculators (1 for OVMS repository, 1-N inside mediapipe) -// for the one inside OVMS repo it makes sense to reuse code from ovms lib -namespace mediapipe { -using std::endl; - -namespace { -#define ASSERT_CAPI_STATUS_NULL(C_API_CALL) \ - { \ - auto* err = C_API_CALL; \ - if (err != nullptr) { \ - uint32_t code = 0; \ - const char* msg = nullptr; \ - OVMS_StatusCode(err, &code); \ - OVMS_StatusDetails(err, &msg); \ - LOG(INFO) << "Error encountred in OVMSCalculator:" << msg << " code: " << code; \ - OVMS_StatusDelete(err); \ - RET_CHECK(err == nullptr); \ - } \ - } -#define CREATE_GUARD(GUARD_NAME, CAPI_TYPE, CAPI_PTR) \ - std::unique_ptr GUARD_NAME(CAPI_PTR, &(CAPI_TYPE##Delete)); - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunused-function" -static ov::element::Type_t CAPI2OVPrecision(OVMS_DataType datatype) { - static std::unordered_map precisionMap{ - {OVMS_DATATYPE_FP64, ov::element::Type_t::f64}, - {OVMS_DATATYPE_FP32, ov::element::Type_t::f32}, - {OVMS_DATATYPE_FP16, ov::element::Type_t::f16}, - {OVMS_DATATYPE_I64, ov::element::Type_t::i64}, - {OVMS_DATATYPE_I32, ov::element::Type_t::i32}, - {OVMS_DATATYPE_I16, ov::element::Type_t::i16}, - {OVMS_DATATYPE_I8, ov::element::Type_t::i8}, - {OVMS_DATATYPE_I4, ov::element::Type_t::i4}, - {OVMS_DATATYPE_U64, ov::element::Type_t::u64}, - {OVMS_DATATYPE_U32, ov::element::Type_t::u32}, - {OVMS_DATATYPE_U16, ov::element::Type_t::u16}, - {OVMS_DATATYPE_U8, ov::element::Type_t::u8}, - {OVMS_DATATYPE_U4, ov::element::Type_t::u4}, - {OVMS_DATATYPE_U1, ov::element::Type_t::u1}, - {OVMS_DATATYPE_BOOL, ov::element::Type_t::boolean}, - {OVMS_DATATYPE_BF16, ov::element::Type_t::bf16}, - {OVMS_DATATYPE_UNDEFINED, ov::element::Type_t::undefined}, - {OVMS_DATATYPE_DYNAMIC, ov::element::Type_t::dynamic} - // {OVMS_DATATYPE_MIXED, ov::element::Type_t::MIXED}, - // {OVMS_DATATYPE_Q78, ov::element::Type_t::Q78}, - // {OVMS_DATATYPE_BIN, ov::element::Type_t::BIN}, - // {OVMS_DATATYPE_CUSTOM, ov::element::Type_t::CUSTOM - }; - auto it = precisionMap.find(datatype); - if (it == precisionMap.end()) { - return ov::element::Type_t::undefined; - } - return it->second; -} - -static OVMS_DataType OVPrecision2CAPI(ov::element::Type_t datatype) { - static std::unordered_map precisionMap{ - {ov::element::Type_t::f64, OVMS_DATATYPE_FP64}, - {ov::element::Type_t::f32, OVMS_DATATYPE_FP32}, - {ov::element::Type_t::f16, OVMS_DATATYPE_FP16}, - {ov::element::Type_t::i64, OVMS_DATATYPE_I64}, - {ov::element::Type_t::i32, OVMS_DATATYPE_I32}, - {ov::element::Type_t::i16, OVMS_DATATYPE_I16}, - {ov::element::Type_t::i8, OVMS_DATATYPE_I8}, - {ov::element::Type_t::i4, OVMS_DATATYPE_I4}, - {ov::element::Type_t::u64, OVMS_DATATYPE_U64}, - {ov::element::Type_t::u32, OVMS_DATATYPE_U32}, - {ov::element::Type_t::u16, OVMS_DATATYPE_U16}, - {ov::element::Type_t::u8, OVMS_DATATYPE_U8}, - {ov::element::Type_t::u4, OVMS_DATATYPE_U4}, - {ov::element::Type_t::u1, OVMS_DATATYPE_U1}, - {ov::element::Type_t::boolean, OVMS_DATATYPE_BOOL}, - {ov::element::Type_t::bf16, OVMS_DATATYPE_BF16}, - {ov::element::Type_t::undefined, OVMS_DATATYPE_UNDEFINED}, - {ov::element::Type_t::dynamic, OVMS_DATATYPE_DYNAMIC} - // {ov::element::Type_t::, OVMS_DATATYPE_MIXEDMIXED}, - // {ov::element::Type_t::, OVMS_DATATYPE_Q78Q78}, - // {ov::element::Type_t::, OVMS_DATATYPE_BINBIN}, - // {ov::element::Type_t::, OVMS_DATATYPE_CUSTOMCUSTOM - }; - auto it = precisionMap.find(datatype); - if (it == precisionMap.end()) { - return OVMS_DATATYPE_UNDEFINED; - } - return it->second; -} - -static ov::Tensor* makeOvTensor(OVMS_DataType datatype, const int64_t* shape, uint32_t dimCount, const void* voutputData, size_t bytesize) { - ov::Shape ovShape; - for (size_t i = 0; i < dimCount; ++i) { - ovShape.push_back(shape[i]); - } - // here we make copy of underlying OVMS repsonse tensor - ov::Tensor* output = new ov::Tensor(CAPI2OVPrecision(datatype), ovShape); - std::memcpy(output->data(), voutputData, bytesize); - return output; -} -static ov::Tensor makeOvTensorO(OVMS_DataType datatype, const int64_t* shape, uint32_t dimCount, const void* voutputData, size_t bytesize) { - ov::Shape ovShape; - for (size_t i = 0; i < dimCount; ++i) { - ovShape.push_back(shape[i]); - } - // here we make copy of underlying OVMS repsonse tensor - ov::Tensor output(CAPI2OVPrecision(datatype), ovShape); - std::memcpy(output.data(), voutputData, bytesize); - return output; -} -} // namespace - -class OVMSOVCalculator : public CalculatorBase { - OVMS_Server* cserver{nullptr}; - OVMS_ServerSettings* _serverSettings{nullptr}; - OVMS_ModelsSettings* _modelsSettings{nullptr}; - std::unordered_map outputNameToTag; - -public: - static absl::Status GetContract(CalculatorContract* cc) { - RET_CHECK(!cc->Inputs().GetTags().empty()); - RET_CHECK(!cc->Outputs().GetTags().empty()); - for (const std::string& tag : cc->Inputs().GetTags()) { - cc->Inputs().Tag(tag).Set(); - } - for (const std::string& tag : cc->Outputs().GetTags()) { - cc->Outputs().Tag(tag).Set(); - } - const auto& options = cc->Options(); - RET_CHECK(!options.servable_name().empty()); - // TODO validate version from string - // TODO validate service url format - RET_CHECK(options.config_path().empty() || - options.service_url().empty()); - // TODO validate tag_to_tensor maps so that key fulfill regex - return absl::OkStatus(); - } - - absl::Status Close(CalculatorContext* cc) final { - const auto& options = cc->Options(); - if (!options.config_path().empty()) { - OVMS_ModelsSettingsDelete(_modelsSettings); - OVMS_ServerSettingsDelete(_serverSettings); - // Close is called on input node and output node in initial pipeline - // Commented out since for now this happens twice in 2 nodes graph. Server will close - // OVMS_ServerDelete(cserver); TODO this should happen onlif graph is used once - // moreover we may need several ovms calculators use in graph each providing its own model? how to handle then different model inputs, as wel as config? - } - return absl::OkStatus(); - } - absl::Status Open(CalculatorContext* cc) final { - for (CollectionItemId id = cc->Inputs().BeginId(); id < cc->Inputs().EndId(); ++id) { - if (!cc->Inputs().Get(id).Header().IsEmpty()) { - cc->Outputs().Get(id).SetHeader(cc->Inputs().Get(id).Header()); - } - } - if (cc->OutputSidePackets().NumEntries() != 0) { - for (CollectionItemId id = cc->InputSidePackets().BeginId(); id < cc->InputSidePackets().EndId(); ++id) { - cc->OutputSidePackets().Get(id).Set(cc->InputSidePackets().Get(id)); - } - } - cc->SetOffset(TimestampDiff(0)); - - const auto& options = cc->Options(); - OVMS_ServerNew(&cserver); - if (!options.config_path().empty()) { - OVMS_ServerSettingsNew(&_serverSettings); - OVMS_ModelsSettingsNew(&_modelsSettings); - OVMS_ModelsSettingsSetConfigPath(_modelsSettings, options.config_path().c_str()); - OVMS_ServerSettingsSetLogLevel(_serverSettings, OVMS_LOG_DEBUG); - OVMS_ServerStartFromConfigurationFile(cserver, _serverSettings, _modelsSettings); - } - for (const auto& [key, value] : options.tag_to_output_tensor_names()) { - outputNameToTag[value] = key; - } - return absl::OkStatus(); - } - - absl::Status Process(CalculatorContext* cc) final { - const auto& options = cc->Options(); - ///////////////////// - // PREPARE REQUEST - ///////////////////// - OVMS_InferenceRequest* request{nullptr}; - auto servableVersionOpt = ovms::stou32(options.servable_version().c_str()); - uint64_t servableVersion = servableVersionOpt.value_or(0); - ASSERT_CAPI_STATUS_NULL(OVMS_InferenceRequestNew(&request, cserver, options.servable_name().c_str(), servableVersion)); - CREATE_GUARD(requestGuard, OVMS_InferenceRequest, request); - - // PREPARE EACH INPUT - // extract single tensor - const auto inputTagInputMap = options.tag_to_input_tensor_names(); - const auto inputTagOutputMap = options.tag_to_output_tensor_names(); - for (const std::string& tag : cc->Inputs().GetTags()) { - // TODO validate existence of tag key in map - const char* realInputName = inputTagInputMap.at(tag).c_str(); - - auto& packet = cc->Inputs().Tag(tag).Get(); - ov::Tensor input_tensor(packet); - const float* input_tensor_access = reinterpret_cast(input_tensor.data()); - std::stringstream ss; - ss << "Calculator received tensor: [ "; - for (int x = 0; x < 10; ++x) { - ss << input_tensor_access[x] << " "; - } - ss << " ] timestamp: " << cc->InputTimestamp().DebugString(); - LOG(INFO) << ss.str(); - const auto& ovInputShape = input_tensor.get_shape(); - std::vector inputShape(ovInputShape.begin(), ovInputShape.end()); // TODO ensure ov tensors shapes conversions return error in all calcs - OVMS_DataType inputDataType = OVPrecision2CAPI(input_tensor.get_element_type()); - ASSERT_CAPI_STATUS_NULL(OVMS_InferenceRequestAddInput(request, realInputName, inputDataType, inputShape.data(), inputShape.size())); - const uint32_t notUsedNum = 0; - // TODO handle hardcoded buffertype, notUsedNum additional options? side packets? - ASSERT_CAPI_STATUS_NULL(OVMS_InferenceRequestInputSetData(request, - realInputName, - reinterpret_cast(input_tensor.data()), - input_tensor.get_byte_size(), - OVMS_BUFFERTYPE_CPU, - notUsedNum)); - } - ////////////////// - // INFERENCE - ////////////////// - OVMS_InferenceResponse* response = nullptr; - ASSERT_CAPI_STATUS_NULL(OVMS_Inference(cserver, request, &response)); - CREATE_GUARD(responseGuard, OVMS_InferenceResponse, response); - // verify GetOutputCount - uint32_t outputCount = 42; - ASSERT_CAPI_STATUS_NULL(OVMS_InferenceResponseOutputCount(response, &outputCount)); - RET_CHECK(outputCount == cc->Outputs().GetTags().size()); - uint32_t parameterCount = 42; - ASSERT_CAPI_STATUS_NULL(OVMS_InferenceResponseParameterCount(response, ¶meterCount)); - // TODO handle output filtering. Graph definition could suggest - // that we are not interested in all outputs from OVMS Inference - const void* voutputData; - size_t bytesize = 42; - OVMS_DataType datatype = (OVMS_DataType)199; - const int64_t* shape{nullptr}; - size_t dimCount = 42; - OVMS_BufferType bufferType = (OVMS_BufferType)199; - uint32_t deviceId = 42; - const char* outputName{nullptr}; - for (size_t i = 0; i < outputCount; ++i) { - ASSERT_CAPI_STATUS_NULL(OVMS_InferenceResponseOutput(response, i, &outputName, &datatype, &shape, &dimCount, &voutputData, &bytesize, &bufferType, &deviceId)); - ov::Tensor* outOvTensor = makeOvTensor(datatype, shape, dimCount, voutputData, bytesize); - cc->Outputs().Tag(outputNameToTag.at(outputName)).Add(outOvTensor, cc->InputTimestamp()); - } - return absl::OkStatus(); - } -}; -#pragma GCC diagnostic pop -REGISTER_CALCULATOR(OVMSOVCalculator); -} // namespace mediapipe diff --git a/mediapipe/calculators/ovms/ovmscalculator.proto b/mediapipe/calculators/ovms/ovmscalculator.proto deleted file mode 100644 index 3d2920560a..0000000000 --- a/mediapipe/calculators/ovms/ovmscalculator.proto +++ /dev/null @@ -1,49 +0,0 @@ -//***************************************************************************** -// Copyright 2023 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** - -syntax = "proto2"; -package mediapipe; - -import "mediapipe/framework/calculator.proto"; - -message OVMSCalculatorOptions { - extend mediapipe.CalculatorOptions { - // https://github.com/google/mediapipe/issues/634 have to be unique in app - // no rule to obtain this - optional OVMSCalculatorOptions ext = 113473742; - } - required string servable_name = 1; - // [1-9][0-9]* - required string servable_version = 2; // TODO decide if follow KFS proto or use int - // keep mappings separately in case input/output tensor have the same name -/* - tag_to_input_tensor_names {$ - key: "INPUT"$ - value: "input_tensor/input_features:0"$ - }$ - tag_to_input_tensor_names {$ - key: "DATA"$ - value: "data_tensor"$ - }$ -*/ - map tag_to_input_tensor_names = 3; - map tag_to_output_tensor_names = 4; - // service_url: "13.21.212.171:9718" - optional string service_url = 5; - // config_path: "/models/config.json" - // TODO decide if it should be at all possible since OVMS does not have a C-API way to stop and then restart - optional string config_path = 6; -} diff --git a/mediapipe/calculators/ovms/test_data/config.json b/mediapipe/calculators/ovms/test_data/config.json deleted file mode 100644 index 9c76c669e5..0000000000 --- a/mediapipe/calculators/ovms/test_data/config.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "model_config_list" : [ - { - "config": { - "name": "dummy", - "base_path": "./dummy" - } - } - ] -} diff --git a/mediapipe/calculators/ovms/test_data/dummy/1/dummy.bin b/mediapipe/calculators/ovms/test_data/dummy/1/dummy.bin deleted file mode 100644 index 9c12e8adf98ba3c9bdad8481352f1dac07f3df52..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4 LcmZQzXs`zW0Yd=5 diff --git a/mediapipe/calculators/ovms/test_data/dummy/1/dummy.xml b/mediapipe/calculators/ovms/test_data/dummy/1/dummy.xml deleted file mode 100644 index 8df1493123..0000000000 --- a/mediapipe/calculators/ovms/test_data/dummy/1/dummy.xml +++ /dev/null @@ -1,99 +0,0 @@ - - - - - - - - 1 - 10 - - - - - - - - 1 - 1 - - - - - - - 1 - 10 - - - 1 - 1 - - - - - 1 - 10 - - - - - - - 1 - 10 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/mediapipe/examples/desktop/hello_ovms/BUILD b/mediapipe/examples/desktop/hello_ovms/BUILD deleted file mode 100644 index f27c17810a..0000000000 --- a/mediapipe/examples/desktop/hello_ovms/BUILD +++ /dev/null @@ -1,45 +0,0 @@ -# -# Copyright (c) 2023 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -licenses(["notice"]) - -package(default_visibility = ["//visibility:public"]) - -cc_binary( - name = "hello_ovms", - srcs = [ - "hello_ovms.cc", - "c_api_test_utils.hpp" - ], - deps = [ - "@ovms//src:ovms_lib", - "//mediapipe/calculators/ovms:ovms_calculator", - "//mediapipe/framework:calculator_graph", - "//mediapipe/framework/port:logging", - "//mediapipe/framework/port:parse_text_proto", - "//mediapipe/framework/port:status", - ], - linkopts = [ - "-lxml2", - "-luuid", - "-lstdc++fs", - "-lcrypto", - ], - copts = [ - "-Iexternal/ovms/src", - "-Iexternal/ovms/src/test", - ], -) - diff --git a/mediapipe/examples/desktop/hello_ovms/add_two_inputs_model/1/add.bin b/mediapipe/examples/desktop/hello_ovms/add_two_inputs_model/1/add.bin deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/mediapipe/examples/desktop/hello_ovms/add_two_inputs_model/1/add.xml b/mediapipe/examples/desktop/hello_ovms/add_two_inputs_model/1/add.xml deleted file mode 100644 index a901c2e546..0000000000 --- a/mediapipe/examples/desktop/hello_ovms/add_two_inputs_model/1/add.xml +++ /dev/null @@ -1,91 +0,0 @@ - - - - - - - - 1 - 10 - - - - - - - - 1 - 10 - - - - - - - 1 - 10 - - - 1 - 10 - - - - - 1 - 10 - - - - - - - 1 - 10 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/mediapipe/examples/desktop/hello_ovms/c_api_test_utils.hpp b/mediapipe/examples/desktop/hello_ovms/c_api_test_utils.hpp deleted file mode 100644 index 7df9a2cc8b..0000000000 --- a/mediapipe/examples/desktop/hello_ovms/c_api_test_utils.hpp +++ /dev/null @@ -1,75 +0,0 @@ -//***************************************************************************** -// Copyright 2023 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once -#include - -#define EXPECT_EQ1(expr1, expr2, errmsg) \ - { \ - if (expr1 != expr2) { \ - std::cout << "EXPECT_EQ1 ERROR:" << errmsg << std::endl; \ - exit(1); \ - } \ - } \ - -#define EXPECT_NEQ1(expr1, expr2, errmsg) \ - { \ - if (expr1 == expr2) { \ - std::cout << "EXPECT_NEQ1 ERROR: "errmsg << std::endl; \ - exit(1); \ - } \ - } - -#define ASSERT_CAPI_STATUS_NULL(C_API_CALL) \ - { \ - auto* err = C_API_CALL; \ - if (err != nullptr) { \ - uint32_t code = 0; \ - const char* msg = nullptr; \ - OVMS_StatusCode(err, &code); \ - OVMS_StatusDetails(err, &msg); \ - std::string smsg(msg); \ - OVMS_StatusDelete(err); \ - EXPECT_EQ1(0, code, smsg); \ - EXPECT_EQ1(err, nullptr, smsg) \ - } \ - } - -#define ASSERT_CAPI_STATUS_NOT_NULL(C_API_CALL) \ - { \ - auto* err = C_API_CALL; \ - if (err != nullptr) { \ - OVMS_StatusDelete(err); \ - } else { \ - EXPECT_NEQ1(err, nullptr, "ERROR: OVMS_StatusDelete(err)"); \ - } \ - } - -#define ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(C_API_CALL, EXPECTED_STATUS_CODE) \ - { \ - auto* err = C_API_CALL; \ - if (err != nullptr) { \ - uint32_t code = 0; \ - const char* details = nullptr; \ - EXPECT_EQ1(OVMS_StatusCode(err, &code), nullptr,"OVMS_StatusCode"); \ - EXPECT_EQ1(OVMS_StatusDetails(err, &details), nullptr); \ - EXPECT_NEQ1(details, nullptr, "details error"); \ - EXPECT_EQ1(code, static_cast(EXPECTED_STATUS_CODE), \ - std::string{"wrong code: "} + std::to_string(code) + std::string{"; details: "}); \ - OVMS_StatusDelete(err); \ - } else { \ - EXPECT_NEQ1(err, nullptr,""); \ - } \ - } diff --git a/mediapipe/examples/desktop/hello_ovms/config.json b/mediapipe/examples/desktop/hello_ovms/config.json deleted file mode 100644 index 0e1d673c66..0000000000 --- a/mediapipe/examples/desktop/hello_ovms/config.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "model_config_list": [ - {"config": { - "name": "add", - "base_path": "/mediapipe/mediapipe/examples/desktop/hello_ovms/add_two_inputs_model", - "shape": {"input1":"(1, 10)"} - } - } - ] -} \ No newline at end of file diff --git a/mediapipe/examples/desktop/hello_ovms/hello_ovms.cc b/mediapipe/examples/desktop/hello_ovms/hello_ovms.cc deleted file mode 100644 index a79a7d51f3..0000000000 --- a/mediapipe/examples/desktop/hello_ovms/hello_ovms.cc +++ /dev/null @@ -1,149 +0,0 @@ -// -// Copyright (c) 2023 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "ovms.h" -#include "c_api_test_utils.hpp" -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wdeprecated-declarations" -#include "mediapipe/framework/calculator_graph.h" -#include "mediapipe/framework/port/logging.h" -#include "mediapipe/framework/port/parse_text_proto.h" -#include "mediapipe/framework/port/status.h" -#pragma GCC diagnostic pop - -#include - -#include -using std::cout; -using std::endl; -namespace mediapipe { - -#define CALL_AND_CHECK_RET(CALL) \ - { \ - auto absStatus = CALL; \ - if (!absStatus.ok()) { \ - cout << __FILE__ << ":" << __LINE__ << endl; \ - const std::string absMessage = absStatus.ToString(); \ - std::cout << "ERROR when calling: " << #CALL << " ERROR: " << std::move(absMessage)<< std::endl; \ - exit(1); \ - } \ - } \ - -absl::Status RunMediapipeGraph() { - // Configures a simple graph, which concatenates 2 PassThroughCalculators. - CalculatorGraphConfig config = - ParseTextProtoOrDie(R"pb( - input_stream: "in1" - input_stream: "in2" - output_stream: "out" - node { - calculator: "OpenVINOModelServerSessionCalculator" - output_side_packet: "SESSION:session" - node_options: { - [type.googleapis.com / mediapipe.OpenVINOModelServerSessionCalculatorOptions]: { - servable_name: "add" - servable_version: "1" - } - } - } - node { - calculator: "OpenVINOInferenceCalculator" - input_side_packet: "SESSION:session" - input_stream: "INPUT1:in1" - input_stream: "INPUT2:in2" - output_stream: "SUM:out" - node_options: { - [type.googleapis.com / mediapipe.OpenVINOInferenceCalculatorOptions]: { - tag_to_input_tensor_names { - key: "INPUT1" - value: "input1" - } - tag_to_input_tensor_names { - key: "INPUT2" - value: "input2" - } - tag_to_output_tensor_names { - key: "SUM" - value: "sum" - } - } - } - } - )pb"); - - CalculatorGraph graph; - CALL_AND_CHECK_RET(graph.Initialize(config)); - - ASSIGN_OR_RETURN(OutputStreamPoller poller, - graph.AddOutputStreamPoller("out")); - CALL_AND_CHECK_RET(graph.StartRun({})); - - ov::Tensor tensor1 = ov::Tensor(ov::element::f32, {1,10}); - float data[] = {1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0}; - std::memcpy(tensor1.data(), data, tensor1.get_byte_size()); - CALL_AND_CHECK_RET(graph.AddPacketToInputStream( - "in1", MakePacket(std::move(tensor1)).At(Timestamp(0)))); - - - ov::Tensor tensor2 = ov::Tensor(ov::element::f32, {1,10}); - float data2[] = {10.0, 9.0, 8.0, 7.0, 6.0, 5.0, 4.0, 3.0, 2.0, 1.0}; - std::memcpy(tensor2.data(), data2, tensor2.get_byte_size()); - CALL_AND_CHECK_RET(graph.AddPacketToInputStream( - "in2", MakePacket(std::move(tensor2)).At(Timestamp(0)))); - - - // Close the input stream "in". - CALL_AND_CHECK_RET(graph.CloseInputStream("in1")); - CALL_AND_CHECK_RET(graph.CloseInputStream("in2")); - mediapipe::Packet packet; - // Get the output packets string. - while (poller.Next(&packet)) { - ov::Tensor tensor3 = packet.Get(); - float* ptr = reinterpret_cast(tensor3.data()); - for(int i = 0; i < 10; i++){ - cout << "Output tensor data: "<< i << " - " << ptr[i] << std::endl; - } - } - return graph.WaitUntilDone(); -} -} // namespace mediapipe - -void InitOvmsServer(OVMS_Server* srv, OVMS_ServerSettings* serverSettings, OVMS_ModelsSettings* modelsSettings) -{ - ASSERT_CAPI_STATUS_NULL(OVMS_ServerNew(&srv)); - ASSERT_CAPI_STATUS_NULL(OVMS_ServerSettingsNew(&serverSettings)); - ASSERT_CAPI_STATUS_NULL(OVMS_ServerSettingsSetLogLevel(serverSettings, OVMS_LOG_DEBUG)); - ASSERT_CAPI_STATUS_NULL(OVMS_ModelsSettingsNew(&modelsSettings)); - - ASSERT_CAPI_STATUS_NULL(OVMS_ModelsSettingsSetConfigPath(modelsSettings, "/mediapipe/mediapipe/examples/desktop/hello_ovms/config.json")); - ASSERT_CAPI_STATUS_NULL(OVMS_ServerStartFromConfigurationFile(srv, serverSettings, modelsSettings)); -} - -int main(int argc, char** argv) { - google::InitGoogleLogging(argv[0]); - - OVMS_Server* srv = nullptr; - OVMS_ServerSettings* serverSettings = nullptr; - OVMS_ModelsSettings* modelsSettings = nullptr; - - InitOvmsServer(srv, serverSettings, modelsSettings); - - CALL_AND_CHECK_RET(mediapipe::RunMediapipeGraph()); - - OVMS_ModelsSettingsDelete(modelsSettings); - OVMS_ServerSettingsDelete(serverSettings); - OVMS_ServerDelete(srv); - return 0; -} diff --git a/mediapipe/graphs/object_detection/object_detection_desktop_openvino_graph.pbtxt b/mediapipe/graphs/object_detection/object_detection_desktop_openvino_graph.pbtxt deleted file mode 100644 index 2301e68a46..0000000000 --- a/mediapipe/graphs/object_detection/object_detection_desktop_openvino_graph.pbtxt +++ /dev/null @@ -1,181 +0,0 @@ -# MediaPipe graph that performs object detection on desktop with Openvino -# on CPU. -# Used in the example in -# mediapipe/examples/desktop/object_detection:object_detection_openvino. - -# max_queue_size limits the number of packets enqueued on any input stream -# by throttling inputs to the graph. This makes the graph only process one -# frame per time. -max_queue_size: 1 - -# Decodes an input video file into images and a video header. -node { - calculator: "OpenCvVideoDecoderCalculator" - input_side_packet: "INPUT_FILE_PATH:input_video_path" - output_stream: "VIDEO:input_video" - output_stream: "VIDEO_PRESTREAM:input_video_header" -} - -# Transforms the input image on CPU to a 320x320 image. To scale the image, by -# default it uses the STRETCH scale mode that maps the entire input image to the -# entire transformed image. As a result, image aspect ratio may be changed and -# objects in the image may be deformed (stretched or squeezed), but the object -# detection model used in this graph is agnostic to that deformation. -node: { - calculator: "ImageTransformationCalculator" - input_stream: "IMAGE:input_video" - output_stream: "IMAGE:transformed_input_video" - node_options: { - [type.googleapis.com/mediapipe.ImageTransformationCalculatorOptions] { - output_width: 320 - output_height: 320 - } - } -} - -# Converts the transformed input image on CPU into an image tensor as a -# OpenVINOTensor. The zero_center option is set to true to normalize the -# pixel values to [-1.f, 1.f] as opposed to [0.f, 1.f]. -node { - calculator: "OpenVINOConverterCalculator" - input_stream: "IMAGE:transformed_input_video" - output_stream: "TENSORS:image_tensor" - node_options: { - [type.googleapis.com/mediapipe.OpenVINOConverterCalculatorOptions] { - enable_normalization: true - zero_center: true - } - } -} - -# Runs a Openvino model on CPU that takes an image tensor and outputs a -# vector of tensors representing, for instance, detection boxes/keypoints and -# scores. -node { - calculator: "OpenVINOInferenceCalculator" - input_stream: "TENSORS:image_tensor" - output_stream: "TENSORS:detection_tensors" - node_options: { - [type.googleapis.com/mediapipe.OpenVINOInferenceCalculatorOptions] { - model_path: "/mediapipe/mediapipe/models/ovms/ssdlite_object_detection/1/ssdlite_object_detection.tflite" - } - } -} - -# Generates a single side packet containing a vector of SSD anchors based on -# the specification in the options. -node { - calculator: "SsdAnchorsCalculator" - output_side_packet: "anchors" - node_options: { - [type.googleapis.com/mediapipe.SsdAnchorsCalculatorOptions] { - num_layers: 6 - min_scale: 0.2 - max_scale: 0.95 - input_size_height: 320 - input_size_width: 320 - anchor_offset_x: 0.5 - anchor_offset_y: 0.5 - strides: 16 - strides: 32 - strides: 64 - strides: 128 - strides: 256 - strides: 512 - aspect_ratios: 1.0 - aspect_ratios: 2.0 - aspect_ratios: 0.5 - aspect_ratios: 3.0 - aspect_ratios: 0.3333 - reduce_boxes_in_lowest_layer: true - } - } -} - -# Decodes the detection tensors generated by the Openvino model, based on -# the SSD anchors and the specification in the options, into a vector of -# detections. Each detection describes a detected object. -node { - calculator: "OpenVINOTensorsToDetectionsCalculator" - input_stream: "TENSORS:detection_tensors" - input_side_packet: "ANCHORS:anchors" - output_stream: "DETECTIONS:detections" - node_options: { - [type.googleapis.com/mediapipe.OpenVINOTensorsToDetectionsCalculatorOptions] { - num_classes: 91 - num_boxes: 2034 - num_coords: 4 - ignore_classes: 0 - apply_exponential_on_box_size: true - - x_scale: 10.0 - y_scale: 10.0 - h_scale: 5.0 - w_scale: 5.0 - } - } -} - -# Performs non-max suppression to remove excessive detections. -node { - calculator: "NonMaxSuppressionCalculator" - input_stream: "detections" - output_stream: "filtered_detections" - node_options: { - [type.googleapis.com/mediapipe.NonMaxSuppressionCalculatorOptions] { - min_suppression_threshold: 0.4 - min_score_threshold: 0.6 - max_num_detections: 5 - overlap_type: INTERSECTION_OVER_UNION - } - } -} - -# Maps detection label IDs to the corresponding label text. The label map is -# provided in the label_map_path option. -node { - calculator: "DetectionLabelIdToTextCalculator" - input_stream: "filtered_detections" - output_stream: "output_detections" - node_options: { - [type.googleapis.com/mediapipe.DetectionLabelIdToTextCalculatorOptions] { - label_map_path: "mediapipe/models/ssdlite_object_detection_labelmap.txt" - } - } -} - -# Converts the detections to drawing primitives for annotation overlay. -node { - calculator: "DetectionsToRenderDataCalculator" - input_stream: "DETECTIONS:output_detections" - output_stream: "RENDER_DATA:render_data" - node_options: { - [type.googleapis.com/mediapipe.DetectionsToRenderDataCalculatorOptions] { - thickness: 4.0 - color { r: 255 g: 0 b: 0 } - } - } -} - -# Draws annotations and overlays them on top of the input images. -node { - calculator: "AnnotationOverlayCalculator" - input_stream: "IMAGE:input_video" - input_stream: "render_data" - output_stream: "IMAGE:output_video" -} - -# Encodes the annotated images into a video file, adopting properties specified -# in the input video header, e.g., video framerate. -node { - calculator: "OpenCvVideoEncoderCalculator" - input_stream: "VIDEO:output_video" - input_stream: "VIDEO_PRESTREAM:input_video_header" - input_side_packet: "OUTPUT_FILE_PATH:output_video_path" - node_options: { - [type.googleapis.com/mediapipe.OpenCvVideoEncoderCalculatorOptions]: { - codec: "avc1" - video_format: "mp4" - } - } -} diff --git a/mediapipe/graphs/object_detection/object_detection_desktop_ovms1_graph.pbtxt b/mediapipe/graphs/object_detection/object_detection_desktop_ovms1_graph.pbtxt deleted file mode 100644 index 2a460913e0..0000000000 --- a/mediapipe/graphs/object_detection/object_detection_desktop_ovms1_graph.pbtxt +++ /dev/null @@ -1,206 +0,0 @@ -# MediaPipe graph that performs object detection on desktop with TensorFlow Lite -# on CPU. -# Used in the example in -# mediapipe/examples/desktop/object_detection:object_detection_openvino. - -# max_queue_size limits the number of packets enqueued on any input stream -# by throttling inputs to the graph. This makes the graph only process one -# frame per time. -max_queue_size: 1 - -# Decodes an input video file into images and a video header. -node { - calculator: "OpenCvVideoDecoderCalculator" - input_side_packet: "INPUT_FILE_PATH:input_video_path" - output_stream: "VIDEO:input_video" - output_stream: "VIDEO_PRESTREAM:input_video_header" -} - -# Transforms the input image on CPU to a 320x320 image. To scale the image, by -# default it uses the STRETCH scale mode that maps the entire input image to the -# entire transformed image. As a result, image aspect ratio may be changed and -# objects in the image may be deformed (stretched or squeezed), but the object -# detection model used in this graph is agnostic to that deformation. -node: { - calculator: "ImageTransformationCalculator" - input_stream: "IMAGE:input_video" - output_stream: "IMAGE:transformed_input_video" - node_options: { - [type.googleapis.com/mediapipe.ImageTransformationCalculatorOptions] { - output_width: 320 - output_height: 320 - } - } -} - -# Converts the transformed input image on CPU into an image tensor as a -# OpenVINOTensor. The zero_center option is set to true to normalize the -# pixel values to [-1.f, 1.f] as opposed to [0.f, 1.f]. -node { - calculator: "OpenVINOConverterCalculator" - input_stream: "IMAGE:transformed_input_video" - output_stream: "TENSORS:image_tensor" - node_options: { - [type.googleapis.com/mediapipe.OpenVINOConverterCalculatorOptions] { - enable_normalization: true - zero_center: true - } - } -} - -# Runs a TensorFlow Lite model on CPU that takes an image tensor and outputs a -# vector of tensors representing, for instance, detection boxes/keypoints and -# scores. -node { - calculator: "OpenVINOModelServerSessionCalculator" - output_side_packet: "SESSION:session" - node_options: { - [type.googleapis.com / mediapipe.OpenVINOModelServerSessionCalculatorOptions]: { - servable_name: "ssdlite_object_detection" # servable name inside OVMS - servable_version: "1" - server_config: "mediapipe/calculators/ovms/config.json" - } - } -} -node { - calculator: "OpenVINOInferenceCalculator" - input_side_packet: "SESSION:session" - input_stream: "OVTENSORS:image_tensor" - output_stream: "OVTENSORS2:detection_tensors" - node_options: { - [type.googleapis.com / mediapipe.OpenVINOInferenceCalculatorOptions]: { - input_order_list :["normalized_input_image_tensor"] - output_order_list :["raw_outputs/box_encodings","raw_outputs/class_predictions"] - tag_to_input_tensor_names { - key: "OVTENSORS" - value: "normalized_input_image_tensor" - } - tag_to_output_tensor_names { - key: "OVTENSORS1" - value: "raw_outputs/box_encodings" - } - tag_to_output_tensor_names { - key: "OVTENSORS2" - value: "raw_outputs/class_predictions" - } - } - } -} - -# Generates a single side packet containing a vector of SSD anchors based on -# the specification in the options. -node { - calculator: "SsdAnchorsCalculator" - output_side_packet: "anchors" - node_options: { - [type.googleapis.com/mediapipe.SsdAnchorsCalculatorOptions] { - num_layers: 6 - min_scale: 0.2 - max_scale: 0.95 - input_size_height: 320 - input_size_width: 320 - anchor_offset_x: 0.5 - anchor_offset_y: 0.5 - strides: 16 - strides: 32 - strides: 64 - strides: 128 - strides: 256 - strides: 512 - aspect_ratios: 1.0 - aspect_ratios: 2.0 - aspect_ratios: 0.5 - aspect_ratios: 3.0 - aspect_ratios: 0.3333 - reduce_boxes_in_lowest_layer: true - } - } -} - -# Decodes the detection tensors generated by the TensorFlow Lite model, based on -# the SSD anchors and the specification in the options, into a vector of -# detections. Each detection describes a detected object. -node { - calculator: "OpenVINOTensorsToDetectionsCalculator" - input_stream: "TENSORS:detection_tensors" - input_side_packet: "ANCHORS:anchors" - output_stream: "DETECTIONS:detections" - node_options: { - [type.googleapis.com/mediapipe.OpenVINOTensorsToDetectionsCalculatorOptions] { - num_classes: 91 - num_boxes: 2034 - num_coords: 4 - ignore_classes: 0 - apply_exponential_on_box_size: true - - x_scale: 10.0 - y_scale: 10.0 - h_scale: 5.0 - w_scale: 5.0 - } - } -} - -# Performs non-max suppression to remove excessive detections. -node { - calculator: "NonMaxSuppressionCalculator" - input_stream: "detections" - output_stream: "filtered_detections" - node_options: { - [type.googleapis.com/mediapipe.NonMaxSuppressionCalculatorOptions] { - min_suppression_threshold: 0.4 - min_score_threshold: 0.6 - max_num_detections: 5 - overlap_type: INTERSECTION_OVER_UNION - } - } -} - -# Maps detection label IDs to the corresponding label text. The label map is -# provided in the label_map_path option. -node { - calculator: "DetectionLabelIdToTextCalculator" - input_stream: "filtered_detections" - output_stream: "output_detections" - node_options: { - [type.googleapis.com/mediapipe.DetectionLabelIdToTextCalculatorOptions] { - label_map_path: "mediapipe/models/ssdlite_object_detection_labelmap.txt" - } - } -} - -# Converts the detections to drawing primitives for annotation overlay. -node { - calculator: "DetectionsToRenderDataCalculator" - input_stream: "DETECTIONS:output_detections" - output_stream: "RENDER_DATA:render_data" - node_options: { - [type.googleapis.com/mediapipe.DetectionsToRenderDataCalculatorOptions] { - thickness: 4.0 - color { r: 255 g: 0 b: 0 } - } - } -} - -# Draws annotations and overlays them on top of the input images. -node { - calculator: "AnnotationOverlayCalculator" - input_stream: "IMAGE:input_video" - input_stream: "render_data" - output_stream: "IMAGE:output_video" -} - -# Encodes the annotated images into a video file, adopting properties specified -# in the input video header, e.g., video framerate. -node { - calculator: "OpenCvVideoEncoderCalculator" - input_stream: "VIDEO:output_video" - input_stream: "VIDEO_PRESTREAM:input_video_header" - input_side_packet: "OUTPUT_FILE_PATH:output_video_path" - node_options: { - [type.googleapis.com/mediapipe.OpenCvVideoEncoderCalculatorOptions]: { - codec: "avc1" - video_format: "mp4" - } - } -} diff --git a/mediapipe/graphs/object_detection/object_detection_desktop_ovms2_graph.pbtxt b/mediapipe/graphs/object_detection/object_detection_desktop_ovms2_graph.pbtxt deleted file mode 100644 index 488a7ca306..0000000000 --- a/mediapipe/graphs/object_detection/object_detection_desktop_ovms2_graph.pbtxt +++ /dev/null @@ -1,247 +0,0 @@ -# MediaPipe graph that performs object detection on desktop with TensorFlow Lite -# on CPU. -# Used in the example in -# mediapipe/examples/desktop/object_detection:object_detection_openvino. - -# max_queue_size limits the number of packets enqueued on any input stream -# by throttling inputs to the graph. This makes the graph only process one -# frame per time. -max_queue_size: 1 - -# Decodes an input video file into images and a video header. -node { - calculator: "OpenCvVideoDecoderCalculator" - input_side_packet: "INPUT_FILE_PATH:input_video_path" - output_stream: "VIDEO:input_video" - output_stream: "VIDEO_PRESTREAM:input_video_header" -} - -# Transforms the input image on CPU to a 320x320 image. To scale the image, by -# default it uses the STRETCH scale mode that maps the entire input image to the -# entire transformed image. As a result, image aspect ratio may be changed and -# objects in the image may be deformed (stretched or squeezed), but the object -# detection model used in this graph is agnostic to that deformation. -node: { - calculator: "ImageTransformationCalculator" - input_stream: "IMAGE:input_video" - output_stream: "IMAGE:transformed_input_video" - node_options: { - [type.googleapis.com/mediapipe.ImageTransformationCalculatorOptions] { - output_width: 320 - output_height: 320 - } - } -} - -# Converts the transformed input image on CPU into an image tensor as a -# OpenVINOTensor. The zero_center option is set to true to normalize the -# pixel values to [-1.f, 1.f] as opposed to [0.f, 1.f]. -# TODO FIXME check enable_normalization -# node { -# calculator: "OpenVINOConverterCalculator" -# input_stream: "IMAGE:transformed_input_video" -# output_stream: "TENSORS:image_tensor" -# node_options: { -# [type.googleapis.com/mediapipe.OpenVINOConverterCalculatorOptions] { -# enable_normalization: true -# zero_center: true -# } -# } -#} -node { - calculator: "TfLiteConverterCalculator" - input_stream: "IMAGE:transformed_input_video" - output_stream: "TENSORS:image_tensor" - node_options: { - [type.googleapis.com/mediapipe.TfLiteConverterCalculatorOptions] { - zero_center: true - } - } -} - -# Runs a TensorFlow Lite model on CPU that takes an image tensor and outputs a -# vector of tensors representing, for instance, detection boxes/keypoints and -# scores. -#node { -# calculator: "OpenVINOInferenceCalculator" -# input_stream: "TENSORS:image_tensor" -# output_stream: "TENSORS:detection_tensors" -# node_options: { -# [type.googleapis.com/mediapipe.OpenVINOInferenceCalculatorOptions] { -# model_path: "mediapipe/models/ssdlite_object_detection.xml" -# } -# } -#} -node { - calculator: "OpenVINOModelServerSessionCalculator" - output_side_packet: "SESSION:session" - node_options: { - [type.googleapis.com / mediapipe.OpenVINOModelServerSessionCalculatorOptions]: { - servable_name: "ssdlite_object_detection2" # servable name inside OVMS - servable_version: "1" - server_config: "mediapipe/config.json" - } - } -} -# TODO vector -# TODO read config.json -node { - calculator: "OpenVINOInferenceCalculator" - input_side_packet: "SESSION:session" - input_stream: "TFLITE_TENSORS:image_tensor" - output_stream: "TFLITE_TENSORS2:detection_tensors" - node_options: { - [type.googleapis.com / mediapipe.OpenVINOInferenceCalculatorOptions]: { - tag_to_input_tensor_names { - key: "TFLITE_TENSORS" - value: "normalized_input_image_tensor" - } - tag_to_output_tensor_names { - key: "TFLITE_TENSORS1" - value: "raw_outputs/box_encodings" - } - tag_to_output_tensor_names { - key: "TFLITE_TENSORS2" - value: "raw_outputs/class_predictions" - } - } - } -} - -# Generates a single side packet containing a vector of SSD anchors based on -# the specification in the options. -node { - calculator: "SsdAnchorsCalculator" - output_side_packet: "anchors" - node_options: { - [type.googleapis.com/mediapipe.SsdAnchorsCalculatorOptions] { - num_layers: 6 - min_scale: 0.2 - max_scale: 0.95 - input_size_height: 320 - input_size_width: 320 - anchor_offset_x: 0.5 - anchor_offset_y: 0.5 - strides: 16 - strides: 32 - strides: 64 - strides: 128 - strides: 256 - strides: 512 - aspect_ratios: 1.0 - aspect_ratios: 2.0 - aspect_ratios: 0.5 - aspect_ratios: 3.0 - aspect_ratios: 0.3333 - reduce_boxes_in_lowest_layer: true - } - } -} - -# Decodes the detection tensors generated by the TensorFlow Lite model, based on -# the SSD anchors and the specification in the options, into a vector of -# detections. Each detection describes a detected object. -node { - calculator: "TfLiteTensorsToDetectionsCalculator" - input_stream: "TENSORS:detection_tensors" - input_side_packet: "ANCHORS:anchors" - output_stream: "DETECTIONS:detections" - node_options: { - [type.googleapis.com/mediapipe.TfLiteTensorsToDetectionsCalculatorOptions] { - num_classes: 91 - num_boxes: 2034 - num_coords: 4 - ignore_classes: 0 - apply_exponential_on_box_size: true - - x_scale: 10.0 - y_scale: 10.0 - h_scale: 5.0 - w_scale: 5.0 - } - } -} -#node { -# calculator: "OpenVINOTensorsToDetectionsCalculator" -# input_stream: "TENSORS:detection_tensors" -# input_side_packet: "ANCHORS:anchors" -# output_stream: "DETECTIONS:detections" -# node_options: { -# [type.googleapis.com/mediapipe.OpenVINOTensorsToDetectionsCalculatorOptions] { -# num_classes: 91 -# num_boxes: 2034 -# num_coords: 4 -# ignore_classes: 0 -# apply_exponential_on_box_size: true -# -# x_scale: 10.0 -# y_scale: 10.0 -# h_scale: 5.0 -# w_scale: 5.0 -# } -# } -#} - -# Performs non-max suppression to remove excessive detections. -node { - calculator: "NonMaxSuppressionCalculator" - input_stream: "detections" - output_stream: "filtered_detections" - node_options: { - [type.googleapis.com/mediapipe.NonMaxSuppressionCalculatorOptions] { - min_suppression_threshold: 0.4 - min_score_threshold: 0.6 - max_num_detections: 5 - overlap_type: INTERSECTION_OVER_UNION - } - } -} - -# Maps detection label IDs to the corresponding label text. The label map is -# provided in the label_map_path option. -node { - calculator: "DetectionLabelIdToTextCalculator" - input_stream: "filtered_detections" - output_stream: "output_detections" - node_options: { - [type.googleapis.com/mediapipe.DetectionLabelIdToTextCalculatorOptions] { - label_map_path: "mediapipe/models/ssdlite_object_detection_labelmap.txt" - } - } -} - -# Converts the detections to drawing primitives for annotation overlay. -node { - calculator: "DetectionsToRenderDataCalculator" - input_stream: "DETECTIONS:output_detections" - output_stream: "RENDER_DATA:render_data" - node_options: { - [type.googleapis.com/mediapipe.DetectionsToRenderDataCalculatorOptions] { - thickness: 4.0 - color { r: 255 g: 0 b: 0 } - } - } -} - -# Draws annotations and overlays them on top of the input images. -node { - calculator: "AnnotationOverlayCalculator" - input_stream: "IMAGE:input_video" - input_stream: "render_data" - output_stream: "IMAGE:output_video" -} - -# Encodes the annotated images into a video file, adopting properties specified -# in the input video header, e.g., video framerate. -node { - calculator: "OpenCvVideoEncoderCalculator" - input_stream: "VIDEO:output_video" - input_stream: "VIDEO_PRESTREAM:input_video_header" - input_side_packet: "OUTPUT_FILE_PATH:output_video_path" - node_options: { - [type.googleapis.com/mediapipe.OpenCvVideoEncoderCalculatorOptions]: { - codec: "avc1" - video_format: "mp4" - } - } -} diff --git a/mediapipe/graphs/object_detection/object_detection_desktop_ovms_graph.pbtxt b/mediapipe/graphs/object_detection/object_detection_desktop_ovms_graph.pbtxt deleted file mode 100644 index b95a06ab87..0000000000 --- a/mediapipe/graphs/object_detection/object_detection_desktop_ovms_graph.pbtxt +++ /dev/null @@ -1,184 +0,0 @@ -# MediaPipe graph that performs object detection on desktop with TensorFlow Lite -# on CPU. -# Used in the example in -# mediapipe/examples/desktop/object_detection:object_detection_openvino. - -# max_queue_size limits the number of packets enqueued on any input stream -# by throttling inputs to the graph. This makes the graph only process one -# frame per time. -max_queue_size: 1 - -input_stream: "input_video" -output_stream: "output_video" - -# Transforms the input image on CPU to a 320x320 image. To scale the image, by -# default it uses the STRETCH scale mode that maps the entire input image to the -# entire transformed image. As a result, image aspect ratio may be changed and -# objects in the image may be deformed (stretched or squeezed), but the object -# detection model used in this graph is agnostic to that deformation. -node: { - calculator: "ImageTransformationCalculator" - input_stream: "IMAGE:input_video" - output_stream: "IMAGE:transformed_input" - node_options: { - [type.googleapis.com/mediapipe.ImageTransformationCalculatorOptions] { - output_width: 320 - output_height: 320 - } - } -} - -# Converts the transformed input image on CPU into an image tensor as a -# OpenVINOTensor. The zero_center option is set to true to normalize the -# pixel values to [-1.f, 1.f] as opposed to [0.f, 1.f]. -node { - calculator: "OpenVINOConverterCalculator" - input_stream: "IMAGE:transformed_input" - output_stream: "TENSORS:image_tensor" - node_options: { - [type.googleapis.com/mediapipe.OpenVINOConverterCalculatorOptions] { - enable_normalization: true - zero_center: true - } - } -} - -node { - calculator: "OpenVINOModelServerSessionCalculator" - output_side_packet: "SESSION:session" - node_options: { - [type.googleapis.com / mediapipe.OpenVINOModelServerSessionCalculatorOptions]: { - servable_name: "ssdlite_object_detection" # servable name inside OVMS - servable_version: "1" - server_config: "mediapipe/calculators/ovms/config.json" - } - } -} - -node { - calculator: "OpenVINOInferenceCalculator" - input_side_packet: "SESSION:session" - input_stream: "OVTENSORS:image_tensor" - output_stream: "OVTENSORS2:detection_tensors" - node_options: { - [type.googleapis.com / mediapipe.OpenVINOInferenceCalculatorOptions]: { - input_order_list :["normalized_input_image_tensor"] - output_order_list :["raw_outputs/box_encodings","raw_outputs/class_predictions"] - tag_to_input_tensor_names { - key: "OVTENSORS" - value: "normalized_input_image_tensor" - } - tag_to_output_tensor_names { - key: "OVTENSORS1" - value: "raw_outputs/box_encodings" - } - tag_to_output_tensor_names { - key: "OVTENSORS2" - value: "raw_outputs/class_predictions" - } - } - } -} - -# Generates a single side packet containing a vector of SSD anchors based on -# the specification in the options. -node { - calculator: "SsdAnchorsCalculator" - output_side_packet: "anchors" - node_options: { - [type.googleapis.com/mediapipe.SsdAnchorsCalculatorOptions] { - num_layers: 6 - min_scale: 0.2 - max_scale: 0.95 - input_size_height: 320 - input_size_width: 320 - anchor_offset_x: 0.5 - anchor_offset_y: 0.5 - strides: 16 - strides: 32 - strides: 64 - strides: 128 - strides: 256 - strides: 512 - aspect_ratios: 1.0 - aspect_ratios: 2.0 - aspect_ratios: 0.5 - aspect_ratios: 3.0 - aspect_ratios: 0.3333 - reduce_boxes_in_lowest_layer: true - } - } -} - -# Decodes the detection tensors generated by the TensorFlow Lite model, based on -# the SSD anchors and the specification in the options, into a vector of -# detections. Each detection describes a detected object. -node { - calculator: "OpenVINOTensorsToDetectionsCalculator" - input_stream: "TENSORS:detection_tensors" - input_side_packet: "ANCHORS:anchors" - output_stream: "DETECTIONS:detections" - node_options: { - [type.googleapis.com/mediapipe.OpenVINOTensorsToDetectionsCalculatorOptions] { - num_classes: 91 - num_boxes: 2034 - num_coords: 4 - ignore_classes: 0 - apply_exponential_on_box_size: true - - x_scale: 10.0 - y_scale: 10.0 - h_scale: 5.0 - w_scale: 5.0 - } - } -} - -# Performs non-max suppression to remove excessive detections. -node { - calculator: "NonMaxSuppressionCalculator" - input_stream: "detections" - output_stream: "filtered_detections" - node_options: { - [type.googleapis.com/mediapipe.NonMaxSuppressionCalculatorOptions] { - min_suppression_threshold: 0.4 - min_score_threshold: 0.6 - max_num_detections: 5 - overlap_type: INTERSECTION_OVER_UNION - } - } -} - -# Maps detection label IDs to the corresponding label text. The label map is -# provided in the label_map_path option. -node { - calculator: "DetectionLabelIdToTextCalculator" - input_stream: "filtered_detections" - output_stream: "output_detections" - node_options: { - [type.googleapis.com/mediapipe.DetectionLabelIdToTextCalculatorOptions] { - label_map_path: "mediapipe/models/ssdlite_object_detection_labelmap.txt" - } - } -} - -# Converts the detections to drawing primitives for annotation overlay. -node { - calculator: "DetectionsToRenderDataCalculator" - input_stream: "DETECTIONS:output_detections" - output_stream: "RENDER_DATA:render_data" - node_options: { - [type.googleapis.com/mediapipe.DetectionsToRenderDataCalculatorOptions] { - thickness: 4.0 - color { r: 255 g: 0 b: 0 } - } - } -} - -# Draws annotations and overlays them on top of the input images. -node { - calculator: "AnnotationOverlayCalculator" - input_stream: "IMAGE:transformed_input" - input_stream: "render_data" - output_stream: "IMAGE:output_video" -} \ No newline at end of file diff --git a/mediapipe/models/ovms/config_holistic.json b/mediapipe/models/ovms/config_holistic.json deleted file mode 100644 index 8dcbd5f117..0000000000 --- a/mediapipe/models/ovms/config_holistic.json +++ /dev/null @@ -1,46 +0,0 @@ -{ - "model_config_list" : [ - { "config": { - "name":"face_landmark", - "base_path":"./face_landmark", - "plugin_config": {"PERFORMANCE_HINT": "LATENCY", "AFFINITY": "NUMA"} - } - }, - { "config": { - "name":"hand_landmark_full", - "base_path":"./hand_landmark_full", - "plugin_config": {"PERFORMANCE_HINT": "LATENCY", "AFFINITY": "NUMA"} - } - }, - { "config": { - "name":"hand_recrop", - "base_path":"./hand_recrop", - "plugin_config": {"PERFORMANCE_HINT": "LATENCY", "AFFINITY": "NUMA"} - } - }, - { "config": { - "name":"pose_detection", - "base_path":"./pose_detection", - "plugin_config": {"PERFORMANCE_HINT": "LATENCY", "AFFINITY": "NUMA"} - } - }, - { "config": { - "name":"face_detection", - "base_path":"./face_detection_short_range", - "plugin_config": {"PERFORMANCE_HINT": "LATENCY", "AFFINITY": "NUMA"} - } - }, - { "config": { - "name":"pose_landmark_full", - "base_path":"./pose_landmark_full", - "plugin_config": {"PERFORMANCE_HINT": "LATENCY", "AFFINITY": "NUMA"} - } - }, - { "config": { - "name":"iris_landmark", - "base_path":"./iris_landmark", - "plugin_config": {"PERFORMANCE_HINT": "LATENCY", "AFFINITY": "NUMA"} - } - } - ] -} diff --git a/setup_ovms.py b/setup_ovms.py deleted file mode 100644 index 6795f669e2..0000000000 --- a/setup_ovms.py +++ /dev/null @@ -1,229 +0,0 @@ -# -# Copyright (c) 2023 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import os -import shutil -import subprocess -import sys -import getopt -import traceback - -__version__ = '1.0' - -class SetupOpenvinoModelServer(): - def __init__(self, force): - self.build_lib = "mediapipe/models/ovms" - self.force = force - - def run_command(self, command): - print(command) - if subprocess.call(command.split()) != 0: - sys.exit(-1) - - def get_dst(self, build_lib, file): - dst = os.path.join(build_lib + '/', file.replace("/","/1/")) - dst_dir = os.path.dirname(file) - # Workaround to copy every model in separate directory - model_name = os.path.basename(file).replace(".tflite","") - dir_name = os.path.basename(dst_dir) - - if dir_name != model_name: - dst = dst.replace(dir_name + "/", model_name + "/") - - if 'ssdlite_object_detection' in file: - build_file = os.path.join('mediapipe/', file) - dst = os.path.join(build_lib + '/', file.replace("/","/1/")) - dst = dst.replace('models/1', model_name + '/1') - else: - build_file = os.path.join('mediapipe/modules/', file) - - dst_dir = os.path.dirname(dst) - - return dst, dst_dir, build_file - - def _copy_to_build_lib_dir(self, build_lib, file): - """Copy a file from bazel-bin to the build lib dir.""" - - dst, dst_dir, build_file = self.get_dst(build_lib, file) - if not os.path.exists(dst_dir): - os.makedirs(dst_dir) - print("Copy to: " + dst) - shutil.copyfile(os.path.join('bazel-bin/', build_file), dst) - - def _download_external_file(self, external_file): - """Download an external file from GCS via Bazel.""" - - build_file = os.path.join('mediapipe/modules/', external_file) - if 'ssdlite_object_detection' in external_file: - build_file = os.path.join('mediapipe/', external_file) - fetch_model_command = [ - 'bazel', - 'build', - build_file, - ] - if subprocess.call(fetch_model_command) != 0: - sys.exit(-1) - self._copy_to_build_lib_dir(self.build_lib, external_file) - - def _copy_pbxt_file(self, external_file): - - file_to_copy = os.path.join('mediapipe/modules/', external_file) - - dst = os.path.join(self.build_lib + '/', external_file) - dst_dir = os.path.dirname(external_file) - - if dst_dir == "face_detection": - new_dst_dir = "face_detection_short_range" - dst = dst.replace(dst_dir + "/", new_dst_dir + "/") - - if dst_dir == "pose_landmark": - new_dst_dir = "pose_landmark_full" - dst = dst.replace(dst_dir + "/", new_dst_dir + "/") - - if dst_dir == "hand_landmark": - new_dst_dir = "hand_landmark_full" - dst = dst.replace(dst_dir + "/", new_dst_dir + "/") - - dst_dir = os.path.dirname(dst) - if not os.path.exists(dst_dir): - os.makedirs(dst_dir) - print("Copy to: " + dst) - shutil.copyfile(file_to_copy, dst) - - def convert_pose(self): - dst = "mediapipe/models/ovms/pose_detection/1/pose_detection.tflite" - - if os.path.exists(dst): - if not self.force: - print("File exists , not converting: " + dst + " use --force argument to overwrite.\n") - return - else: - print("Re downloading pose model for conversion.") - self._download_external_file('pose_detection/pose_detection.tflite') - - else: - print("File not downloaded: " + dst + " Run setup_ovms.py --get_models first.") - exit(0) - - print("Converting pose detection model") - self.run_command("cp -r " + dst +" .") - self.run_command("tflite2tensorflow --model_path pose_detection.tflite --flatc_path flatbuffers/build/flatc --schema_path schema.fbs --output_pb") - self.run_command("tflite2tensorflow --model_path pose_detection.tflite --flatc_path flatbuffers/build/flatc --schema_path schema.fbs --output_no_quant_float32_tflite --output_dynamic_range_quant_tflite --output_weight_quant_tflite --output_float16_quant_tflite --output_integer_quant_tflite") - self.run_command("cp -rf saved_model/model_float32.tflite " + dst) - self.run_command("rm -rf pose_detection.tflite") - self.run_command("rm -rf saved_model") - - def get_graphs(self): - external_files = [ - 'face_detection/face_detection.pbtxt', - 'face_landmark/face_landmark_cpu.pbtxt', - 'hand_landmark/hand_landmark_cpu.pbtxt', - #Not needed ?'holistic_landmark/hand_recrop_by_roi_cpu.pbtxt', - 'holistic_landmark/holistic_landmark_cpu.pbtxt', - 'pose_detection/pose_detection_cpu.pbtxt', - 'pose_landmark/pose_landmark_by_roi_cpu.pbtxt', - ] - for elem in external_files: - print('coping file: %s\n' % elem) - self._copy_pbxt_file(elem) - - def get_models(self): - external_files = [ - # Using short range - # 'face_detection/face_detection_full_range_sparse.tflite', - 'face_detection/face_detection_short_range.tflite', - 'face_landmark/face_landmark.tflite', - # Model loading error - # 'face_landmark/face_landmark_with_attention.tflite', - 'hand_landmark/hand_landmark_full.tflite', - # Using full - # 'hand_landmark/hand_landmark_lite.tflite', - 'holistic_landmark/hand_recrop.tflite', - 'iris_landmark/iris_landmark.tflite', - 'palm_detection/palm_detection_full.tflite', - # Using full - # 'palm_detection/palm_detection_lite.tflite', - # Need to use OV version - 'pose_detection/pose_detection.tflite', - 'pose_landmark/pose_landmark_full.tflite', - # Not working - # 'selfie_segmentation/selfie_segmentation.tflite', - # 'selfie_segmentation/selfie_segmentation_landscape.tflite', - 'models/ssdlite_object_detection.tflite', - ] - for elem in external_files: - dst, dst_dir, build_lib = self.get_dst(self.build_lib, elem) - if os.path.exists(dst) and not self.force: - sys.stderr.write("file exists, not downloading: " + dst + " use --force argument to overwrite.\n") - continue - print('downloading file: %s\n' % elem) - self._download_external_file(elem) - -def printUsage(): - """ Prints information about usage of commandline interface """ - - print(""" Usage description: - - Get models required for ovms inference setup - python setup_ovms.py --get_models - - Get graphs used in holistic client example from ovms repository - python setup_ovms.py --get_graphs - - Convert original pose_detection tflite model - workaround for missing op in ov - python setup_ovms.py --convert_pose - """) - - return - -def get_args(argv): - """ Processing commandline """ - - get_graphs_flag = False - get_models_flag = False - convert_pose = False - force = False - try: - opts, vals = getopt.getopt(argv, "", ["force","convert_pose","get_graphs","get_models","help"]) - except getopt.GetoptError: - print("ERROR: unrecognize option/missing argument/value for known option. Use --help to see list of options") - sys.exit(2) - for opt, val in opts: - if opt in ("--help"): - printUsage() - sys.exit(0) - elif opt in ("--get_graphs"): - get_graphs_flag = True - elif opt in ("--get_models"): - get_models_flag = True - elif opt in ("--convert_pose"): - convert_pose = True - elif opt in ("--force"): - force = True - - return get_graphs_flag, get_models_flag, convert_pose, force - -if __name__ == "__main__": - get_graphs_flag, get_models_flag, convert_pose, force = get_args(sys.argv[1:]) - if get_models_flag: - SetupOpenvinoModelServer(force).get_models() - - # Needed to call only on starting ovm holistic demo from ovms repository using ovms server standalone instance - if get_graphs_flag: - SetupOpenvinoModelServer(force).get_graphs() - - if convert_pose: - SetupOpenvinoModelServer(force).convert_pose() diff --git a/test_demos.sh b/test_demos.sh deleted file mode 100755 index f403c6f9e4..0000000000 --- a/test_demos.sh +++ /dev/null @@ -1,4 +0,0 @@ -#/bin/bash -make docker_build -docker run -it mediapipe_ovms:latest make run_demos | tee test_demos.log -cat test_demos.log | grep -a FPS | grep -v echo diff --git a/third_party/model_api/BUILD b/third_party/model_api/BUILD deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/third_party/model_api/model_api.bzl b/third_party/model_api/model_api.bzl deleted file mode 100644 index 7f1fca6812..0000000000 --- a/third_party/model_api/model_api.bzl +++ /dev/null @@ -1,64 +0,0 @@ -# -# Copyright (c) 2023 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -def _impl(repository_ctx): - http_proxy = repository_ctx.os.environ.get("http_proxy", "") - https_proxy = repository_ctx.os.environ.get("https_proxy", "") - # Note we need to escape '{/}' by doubling them due to call to format - build_file_content = """ -load("@rules_foreign_cc//foreign_cc:cmake.bzl", "cmake") -#load("@bazel_skylib//rules:common_settings.bzl", "string_flag") - -visibility = ["//visibility:public"] - -filegroup( - name = "all_srcs", - srcs = glob(["model_api/cpp/**"]), - visibility = ["//visibility:public"], -) - -cmake( - name = "model_api_cmake", - cache_entries = {{ - "CMAKE_POSITION_INDEPENDENT_CODE": "ON", - "OpenVINO_DIR": "/opt/intel/openvino/runtime/cmake", - }}, - env = {{ - "HTTP_PROXY": "{http_proxy}", - "HTTPS_PROXY": "{https_proxy}", - }}, - lib_source = ":all_srcs", - out_static_libs = ["libmodel_api.a"], - tags = ["requires-network"] -) - -cc_library( - name = "model_api", - deps = [ - "@mediapipe//mediapipe/framework/port:opencv_core", - "@linux_openvino//:openvino", - ":model_api_cmake", - ], - visibility = ["//visibility:public"], -) - -""" - repository_ctx.file("BUILD", build_file_content.format(http_proxy=http_proxy, https_proxy=https_proxy)) - -model_api_repository = repository_rule( - implementation = _impl, - local=False, -) From a4391ba0d82657cc3dff85381adff97917af26dd Mon Sep 17 00:00:00 2001 From: Rafal Sapala Date: Mon, 4 Dec 2023 11:09:26 +0100 Subject: [PATCH 2/4] Object detection ovms --- mediapipe/calculators/tensorflow/BUILD | 46 ----------------- .../examples/desktop/object_detection/BUILD | 28 ----------- mediapipe/graphs/object_detection/BUILD | 50 ------------------- 3 files changed, 124 deletions(-) diff --git a/mediapipe/calculators/tensorflow/BUILD b/mediapipe/calculators/tensorflow/BUILD index e9a41cbbd9..2d69486713 100644 --- a/mediapipe/calculators/tensorflow/BUILD +++ b/mediapipe/calculators/tensorflow/BUILD @@ -52,12 +52,6 @@ proto_library( deps = ["//mediapipe/framework:calculator_proto"], ) -proto_library( - name = "ovms_inference_calculator_proto", - srcs = ["ovms_inference_calculator.proto"], - deps = ["//mediapipe/framework:calculator_proto"], -) - proto_library( name = "tensor_squeeze_dimensions_calculator_proto", srcs = ["tensor_squeeze_dimensions_calculator.proto"], @@ -167,13 +161,6 @@ mediapipe_cc_proto_library( deps = [":tensorflow_inference_calculator_proto"], ) -mediapipe_cc_proto_library( - name = "tensorflow_inference_calculator_cc_proto", - srcs = ["tensorflow_inference_calculator.proto"], - cc_deps = ["//mediapipe/framework:calculator_cc_proto"], - deps = [":tensorflow_inference_calculator_proto"], -) - mediapipe_cc_proto_library( name = "tensorflow_session_from_frozen_graph_generator_cc_proto", srcs = ["tensorflow_session_from_frozen_graph_generator.proto"], @@ -473,39 +460,6 @@ cc_library( alwayslink = 1, ) -cc_library( - name = "ovms_inference_calculator", - srcs = ["ovms_inference_calculator.cc"], - deps = [ - ":ovms_inference_calculator_cc_proto", - "@com_google_absl//absl/log:check", - "//mediapipe/framework:timestamp", - "@com_google_absl//absl/base:core_headers", - "@com_google_absl//absl/memory", - "//mediapipe/framework:calculator_context", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/tool:status_util", - "@com_google_absl//absl/strings", - "@com_google_absl//absl/synchronization", - "//mediapipe/framework/deps:clock", - "//mediapipe/framework/port:status", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:map_util", - "//mediapipe/framework:packet", - ] + select({ - "//conditions:default": [ - "@org_tensorflow//tensorflow/core:framework", - ], - "//mediapipe:android": [ - "@org_tensorflow//tensorflow/core:portable_tensorflow_lib_lite", - ], - "//mediapipe:ios": [ - "@org_tensorflow//tensorflow/core:portable_tensorflow_lib", - ], - }), - alwayslink = 1, -) - cc_library( name = "tensorflow_session", hdrs = [ diff --git a/mediapipe/examples/desktop/object_detection/BUILD b/mediapipe/examples/desktop/object_detection/BUILD index 20070d904a..f443f76d46 100644 --- a/mediapipe/examples/desktop/object_detection/BUILD +++ b/mediapipe/examples/desktop/object_detection/BUILD @@ -42,34 +42,6 @@ cc_binary( ], ) -cc_binary( - name = "object_detection_openvino", - data = [ - "//mediapipe/models:ssdlite_object_detection_labelmap.txt", - ], - deps = [ - "//mediapipe/examples/desktop:simple_run_graph_main", - "//mediapipe/graphs/object_detection:desktop_openvino_calculators", - ], -) - -cc_binary( - name = "object_detection_ovms", - data = [ - "//mediapipe/models:ssdlite_object_detection_labelmap.txt", - ], - deps = [ - "@ovms//src:ovms_lib", - "//mediapipe/examples/desktop:demo_run_graph_main", - "//mediapipe/graphs/object_detection:desktop_ovms_calculators", - # here we declare dependencies on tflite as we can support 2 types - # of graphs: - # 1) using tflite converters & tensors as packet streams - # 2) using openvino converters & tensors as packet streams - #"//mediapipe/graphs/object_detection:desktop_tflite_calculators", - ], -) - cc_binary( name = "object_detection_cpu", data = [ diff --git a/mediapipe/graphs/object_detection/BUILD b/mediapipe/graphs/object_detection/BUILD index 4c7b28dd1d..398fe644f0 100644 --- a/mediapipe/graphs/object_detection/BUILD +++ b/mediapipe/graphs/object_detection/BUILD @@ -76,56 +76,6 @@ cc_library( ], ) -cc_library( - name = "desktop_openvino_calculators", - deps = [ - "//mediapipe/calculators/core:concatenate_vector_calculator", - "//mediapipe/calculators/core:flow_limiter_calculator", - "//mediapipe/calculators/core:previous_loopback_calculator", - "//mediapipe/calculators/core:split_vector_calculator", - "//mediapipe/calculators/image:image_transformation_calculator", - "//mediapipe/calculators/tflite:ssd_anchors_calculator", - "//mediapipe/calculators/openvino:openvino_converter_calculator", - "//mediapipe/calculators/openvino:openvino_inference_calculator", - "//mediapipe/calculators/openvino:openvino_tensors_to_detections_calculator", - "//mediapipe/calculators/util:annotation_overlay_calculator", - "//mediapipe/calculators/util:detection_label_id_to_text_calculator", - "//mediapipe/calculators/util:detections_to_render_data_calculator", - "//mediapipe/calculators/util:non_max_suppression_calculator", - "//mediapipe/calculators/video:opencv_video_decoder_calculator", - "//mediapipe/calculators/video:opencv_video_encoder_calculator", - ], -) - -# TODO for now we use TF Tensor as input so we may want to replace those in the end to pure OV::Tensor converters -cc_library( - name = "desktop_ovms_calculators", - deps = [ -# we should follow more closely MP with tflite convention. for now using the same approach as Nico's demo - #"//mediapipe/calculators/openvino:openvino_converter_calculator", - #"//mediapipe/calculators/openvino:openvino_inference_calculator", - #"//mediapipe/calculators/openvino:openvino_tensors_to_detections_calculator", - - #"//mediapipe/calculators/openvino:openvino_converter_calculator", - #"//mediapipe/calculators/ovms:openvino_inference_calculator", - #"//mediapipe/calculators/openvino:openvino_tensors_to_detections_calculator", -# shared part - "//mediapipe/calculators/ovms:ovms_calculator", # this includes both calculators - "//mediapipe/calculators/core:concatenate_vector_calculator", - "//mediapipe/calculators/core:flow_limiter_calculator", - "//mediapipe/calculators/core:previous_loopback_calculator", - "//mediapipe/calculators/core:split_vector_calculator", - "//mediapipe/calculators/image:image_transformation_calculator", - "//mediapipe/calculators/tflite:ssd_anchors_calculator", - "//mediapipe/calculators/util:annotation_overlay_calculator", - "//mediapipe/calculators/util:detection_label_id_to_text_calculator", - "//mediapipe/calculators/util:detections_to_render_data_calculator", - "//mediapipe/calculators/util:non_max_suppression_calculator", - "//mediapipe/calculators/video:opencv_video_decoder_calculator", - "//mediapipe/calculators/video:opencv_video_encoder_calculator", - ], -) - load( "//mediapipe/framework/tool:mediapipe_graph.bzl", "mediapipe_binary_graph", From 2d04e7ca70a338edcc4d1704b2c4f6584883c119 Mon Sep 17 00:00:00 2001 From: Rafal Sapala Date: Mon, 4 Dec 2023 13:05:01 +0100 Subject: [PATCH 3/4] Changing @ --- docs/development.md | 2 +- mediapipe/modules/face_detection/BUILD | 2 +- mediapipe/modules/face_landmark/BUILD | 2 +- mediapipe/modules/hand_landmark/BUILD | 2 +- mediapipe/modules/holistic_landmark/BUILD | 2 +- mediapipe/modules/iris_landmark/BUILD | 2 +- mediapipe/modules/ovms_modules/BUILD | 10 +++++----- mediapipe/modules/pose_detection/BUILD | 2 +- mediapipe/modules/pose_landmark/BUILD | 2 +- 9 files changed, 13 insertions(+), 13 deletions(-) diff --git a/docs/development.md b/docs/development.md index 4971bbec0b..c0a6dbca40 100644 --- a/docs/development.md +++ b/docs/development.md @@ -19,7 +19,7 @@ python setup_ovms.py --get_models' - modify or create new mediapipe example target with the ```bash -//mediapipe/graphs/object_detection:desktop_ovms_calculators +@mediapipe/graphs/object_detection:desktop_ovms_calculators ``` and ```bash diff --git a/mediapipe/modules/face_detection/BUILD b/mediapipe/modules/face_detection/BUILD index 231d6dba3c..0684b443fa 100644 --- a/mediapipe/modules/face_detection/BUILD +++ b/mediapipe/modules/face_detection/BUILD @@ -111,7 +111,7 @@ mediapipe_simple_subgraph( "//mediapipe/calculators/util:detection_projection_calculator", "//mediapipe/calculators/util:non_max_suppression_calculator", "//mediapipe/calculators/util:to_image_calculator", - "//mediapipe/graphs/object_detection:desktop_ovms_calculators" + "@mediapipe/graphs/object_detection:desktop_ovms_calculators" ], ) diff --git a/mediapipe/modules/face_landmark/BUILD b/mediapipe/modules/face_landmark/BUILD index 45a8522848..ef3351da3b 100644 --- a/mediapipe/modules/face_landmark/BUILD +++ b/mediapipe/modules/face_landmark/BUILD @@ -43,7 +43,7 @@ mediapipe_simple_subgraph( "//mediapipe/calculators/util:landmark_projection_calculator", "//mediapipe/calculators/util:thresholding_calculator", "//mediapipe/framework/tool:switch_container", - "//mediapipe/graphs/object_detection:desktop_ovms_calculators" + "@mediapipe/graphs/object_detection:desktop_ovms_calculators" ], ) diff --git a/mediapipe/modules/hand_landmark/BUILD b/mediapipe/modules/hand_landmark/BUILD index 8cf23474f6..99e92159ee 100644 --- a/mediapipe/modules/hand_landmark/BUILD +++ b/mediapipe/modules/hand_landmark/BUILD @@ -61,7 +61,7 @@ mediapipe_simple_subgraph( "//mediapipe/calculators/util:landmark_projection_calculator", "//mediapipe/calculators/util:thresholding_calculator", "//mediapipe/calculators/util:world_landmark_projection_calculator", - "//mediapipe/graphs/object_detection:desktop_ovms_calculators" + "@mediapipe/graphs/object_detection:desktop_ovms_calculators" ], ) diff --git a/mediapipe/modules/holistic_landmark/BUILD b/mediapipe/modules/holistic_landmark/BUILD index e85da1bd90..d0c0d9068b 100644 --- a/mediapipe/modules/holistic_landmark/BUILD +++ b/mediapipe/modules/holistic_landmark/BUILD @@ -171,7 +171,7 @@ mediapipe_simple_subgraph( "//mediapipe/calculators/util:landmark_projection_calculator", "//mediapipe/calculators/util:landmarks_to_detection_calculator", "//mediapipe/calculators/util:rect_transformation_calculator", - "//mediapipe/graphs/object_detection:desktop_ovms_calculators" + "@mediapipe/graphs/object_detection:desktop_ovms_calculators" ], ) diff --git a/mediapipe/modules/iris_landmark/BUILD b/mediapipe/modules/iris_landmark/BUILD index ad31114f6f..194c16b9f5 100644 --- a/mediapipe/modules/iris_landmark/BUILD +++ b/mediapipe/modules/iris_landmark/BUILD @@ -41,7 +41,7 @@ mediapipe_simple_subgraph( "//mediapipe/calculators/tflite:tflite_tensors_to_landmarks_calculator", "//mediapipe/calculators/util:landmark_letterbox_removal_calculator", "//mediapipe/calculators/util:landmark_projection_calculator", - "//mediapipe/graphs/object_detection:desktop_ovms_calculators", + "@mediapipe/graphs/object_detection:desktop_ovms_calculators", ], ) diff --git a/mediapipe/modules/ovms_modules/BUILD b/mediapipe/modules/ovms_modules/BUILD index 6b53b08ea4..50a94252bb 100644 --- a/mediapipe/modules/ovms_modules/BUILD +++ b/mediapipe/modules/ovms_modules/BUILD @@ -33,7 +33,7 @@ mediapipe_simple_subgraph( graph = "object_detection_ovms.pbtxt", register_as = "ObjectDetectionOvms", deps = [ - "//mediapipe/graphs/object_detection:desktop_ovms_calculators", + "@mediapipe/graphs/object_detection:desktop_ovms_calculators", "@ovms//src:ovms_lib", ], ) @@ -43,7 +43,7 @@ mediapipe_simple_subgraph( graph = "holistic_tracking_ovms.pbtxt", register_as = "HolisticTrackingOvms", deps = [ - "//mediapipe/graphs/object_detection:desktop_ovms_calculators", + "@mediapipe/graphs/object_detection:desktop_ovms_calculators", "@ovms//src:ovms_lib", ], ) @@ -53,7 +53,7 @@ mediapipe_simple_subgraph( graph = "face_detection_ovms.pbtxt", register_as = "FaceDetectionOvms", deps = [ - "//mediapipe/graphs/object_detection:desktop_ovms_calculators", + "@mediapipe/graphs/object_detection:desktop_ovms_calculators", "@ovms//src:ovms_lib", ], ) @@ -63,7 +63,7 @@ mediapipe_simple_subgraph( graph = "iris_tracking_ovms.pbtxt", register_as = "IrisTrackingOvms", deps = [ - "//mediapipe/graphs/object_detection:desktop_ovms_calculators", + "@mediapipe/graphs/object_detection:desktop_ovms_calculators", "@ovms//src:ovms_lib", ], ) @@ -99,7 +99,7 @@ mediapipe_simple_subgraph( graph = "pose_tracking_ovms.pbtxt", register_as = "PoseTrackingOvms", deps = [ - "//mediapipe/graphs/object_detection:desktop_ovms_calculators", + "@mediapipe/graphs/object_detection:desktop_ovms_calculators", ":pose_renderer_cpu", "@ovms//src:ovms_lib", ], diff --git a/mediapipe/modules/pose_detection/BUILD b/mediapipe/modules/pose_detection/BUILD index 87d22c290e..1332e39539 100644 --- a/mediapipe/modules/pose_detection/BUILD +++ b/mediapipe/modules/pose_detection/BUILD @@ -36,7 +36,7 @@ mediapipe_simple_subgraph( "//mediapipe/calculators/tflite:ssd_anchors_calculator", "//mediapipe/calculators/util:detection_letterbox_removal_calculator", "//mediapipe/calculators/util:non_max_suppression_calculator", - "//mediapipe/graphs/object_detection:desktop_ovms_calculators" + "@mediapipe/graphs/object_detection:desktop_ovms_calculators" ], ) diff --git a/mediapipe/modules/pose_landmark/BUILD b/mediapipe/modules/pose_landmark/BUILD index 33a8061fbf..e92da62a80 100644 --- a/mediapipe/modules/pose_landmark/BUILD +++ b/mediapipe/modules/pose_landmark/BUILD @@ -62,7 +62,7 @@ mediapipe_simple_subgraph( "//mediapipe/calculators/image:image_properties_calculator", "//mediapipe/calculators/tensor:image_to_tensor_calculator", "//mediapipe/calculators/tensor:inference_calculator", - "//mediapipe/graphs/object_detection:desktop_ovms_calculators" + "@mediapipe/graphs/object_detection:desktop_ovms_calculators" ], ) From 1e951641df0238c6d31de65375b926d1c0d9b9a2 Mon Sep 17 00:00:00 2001 From: Rafal Sapala Date: Mon, 4 Dec 2023 13:35:03 +0100 Subject: [PATCH 4/4] Proper @ --- docs/development.md | 2 +- mediapipe/modules/face_detection/BUILD | 2 +- mediapipe/modules/face_landmark/BUILD | 2 +- mediapipe/modules/hand_landmark/BUILD | 2 +- mediapipe/modules/holistic_landmark/BUILD | 2 +- mediapipe/modules/iris_landmark/BUILD | 2 +- mediapipe/modules/ovms_modules/BUILD | 10 +++++----- mediapipe/modules/pose_detection/BUILD | 2 +- mediapipe/modules/pose_landmark/BUILD | 2 +- 9 files changed, 13 insertions(+), 13 deletions(-) diff --git a/docs/development.md b/docs/development.md index c0a6dbca40..3f89e488b7 100644 --- a/docs/development.md +++ b/docs/development.md @@ -19,7 +19,7 @@ python setup_ovms.py --get_models' - modify or create new mediapipe example target with the ```bash -@mediapipe/graphs/object_detection:desktop_ovms_calculators +/mediapipe/graphs/object_detection:desktop_ovms_calculators ``` and ```bash diff --git a/mediapipe/modules/face_detection/BUILD b/mediapipe/modules/face_detection/BUILD index 0684b443fa..3c12cfc218 100644 --- a/mediapipe/modules/face_detection/BUILD +++ b/mediapipe/modules/face_detection/BUILD @@ -111,7 +111,7 @@ mediapipe_simple_subgraph( "//mediapipe/calculators/util:detection_projection_calculator", "//mediapipe/calculators/util:non_max_suppression_calculator", "//mediapipe/calculators/util:to_image_calculator", - "@mediapipe/graphs/object_detection:desktop_ovms_calculators" + "@//mediapipe/graphs/object_detection:desktop_ovms_calculators" ], ) diff --git a/mediapipe/modules/face_landmark/BUILD b/mediapipe/modules/face_landmark/BUILD index ef3351da3b..af6a23d1ec 100644 --- a/mediapipe/modules/face_landmark/BUILD +++ b/mediapipe/modules/face_landmark/BUILD @@ -43,7 +43,7 @@ mediapipe_simple_subgraph( "//mediapipe/calculators/util:landmark_projection_calculator", "//mediapipe/calculators/util:thresholding_calculator", "//mediapipe/framework/tool:switch_container", - "@mediapipe/graphs/object_detection:desktop_ovms_calculators" + "@//mediapipe/graphs/object_detection:desktop_ovms_calculators" ], ) diff --git a/mediapipe/modules/hand_landmark/BUILD b/mediapipe/modules/hand_landmark/BUILD index 99e92159ee..5e2a453dc8 100644 --- a/mediapipe/modules/hand_landmark/BUILD +++ b/mediapipe/modules/hand_landmark/BUILD @@ -61,7 +61,7 @@ mediapipe_simple_subgraph( "//mediapipe/calculators/util:landmark_projection_calculator", "//mediapipe/calculators/util:thresholding_calculator", "//mediapipe/calculators/util:world_landmark_projection_calculator", - "@mediapipe/graphs/object_detection:desktop_ovms_calculators" + "@//mediapipe/graphs/object_detection:desktop_ovms_calculators" ], ) diff --git a/mediapipe/modules/holistic_landmark/BUILD b/mediapipe/modules/holistic_landmark/BUILD index d0c0d9068b..126bcfb5f2 100644 --- a/mediapipe/modules/holistic_landmark/BUILD +++ b/mediapipe/modules/holistic_landmark/BUILD @@ -171,7 +171,7 @@ mediapipe_simple_subgraph( "//mediapipe/calculators/util:landmark_projection_calculator", "//mediapipe/calculators/util:landmarks_to_detection_calculator", "//mediapipe/calculators/util:rect_transformation_calculator", - "@mediapipe/graphs/object_detection:desktop_ovms_calculators" + "@//mediapipe/graphs/object_detection:desktop_ovms_calculators" ], ) diff --git a/mediapipe/modules/iris_landmark/BUILD b/mediapipe/modules/iris_landmark/BUILD index 194c16b9f5..1c15b7a68e 100644 --- a/mediapipe/modules/iris_landmark/BUILD +++ b/mediapipe/modules/iris_landmark/BUILD @@ -41,7 +41,7 @@ mediapipe_simple_subgraph( "//mediapipe/calculators/tflite:tflite_tensors_to_landmarks_calculator", "//mediapipe/calculators/util:landmark_letterbox_removal_calculator", "//mediapipe/calculators/util:landmark_projection_calculator", - "@mediapipe/graphs/object_detection:desktop_ovms_calculators", + "@//mediapipe/graphs/object_detection:desktop_ovms_calculators", ], ) diff --git a/mediapipe/modules/ovms_modules/BUILD b/mediapipe/modules/ovms_modules/BUILD index 50a94252bb..f128c0a369 100644 --- a/mediapipe/modules/ovms_modules/BUILD +++ b/mediapipe/modules/ovms_modules/BUILD @@ -33,7 +33,7 @@ mediapipe_simple_subgraph( graph = "object_detection_ovms.pbtxt", register_as = "ObjectDetectionOvms", deps = [ - "@mediapipe/graphs/object_detection:desktop_ovms_calculators", + "@//mediapipe/graphs/object_detection:desktop_ovms_calculators", "@ovms//src:ovms_lib", ], ) @@ -43,7 +43,7 @@ mediapipe_simple_subgraph( graph = "holistic_tracking_ovms.pbtxt", register_as = "HolisticTrackingOvms", deps = [ - "@mediapipe/graphs/object_detection:desktop_ovms_calculators", + "@//mediapipe/graphs/object_detection:desktop_ovms_calculators", "@ovms//src:ovms_lib", ], ) @@ -53,7 +53,7 @@ mediapipe_simple_subgraph( graph = "face_detection_ovms.pbtxt", register_as = "FaceDetectionOvms", deps = [ - "@mediapipe/graphs/object_detection:desktop_ovms_calculators", + "@//mediapipe/graphs/object_detection:desktop_ovms_calculators", "@ovms//src:ovms_lib", ], ) @@ -63,7 +63,7 @@ mediapipe_simple_subgraph( graph = "iris_tracking_ovms.pbtxt", register_as = "IrisTrackingOvms", deps = [ - "@mediapipe/graphs/object_detection:desktop_ovms_calculators", + "@//mediapipe/graphs/object_detection:desktop_ovms_calculators", "@ovms//src:ovms_lib", ], ) @@ -99,7 +99,7 @@ mediapipe_simple_subgraph( graph = "pose_tracking_ovms.pbtxt", register_as = "PoseTrackingOvms", deps = [ - "@mediapipe/graphs/object_detection:desktop_ovms_calculators", + "@//mediapipe/graphs/object_detection:desktop_ovms_calculators", ":pose_renderer_cpu", "@ovms//src:ovms_lib", ], diff --git a/mediapipe/modules/pose_detection/BUILD b/mediapipe/modules/pose_detection/BUILD index 1332e39539..d84bef0a51 100644 --- a/mediapipe/modules/pose_detection/BUILD +++ b/mediapipe/modules/pose_detection/BUILD @@ -36,7 +36,7 @@ mediapipe_simple_subgraph( "//mediapipe/calculators/tflite:ssd_anchors_calculator", "//mediapipe/calculators/util:detection_letterbox_removal_calculator", "//mediapipe/calculators/util:non_max_suppression_calculator", - "@mediapipe/graphs/object_detection:desktop_ovms_calculators" + "@//mediapipe/graphs/object_detection:desktop_ovms_calculators" ], ) diff --git a/mediapipe/modules/pose_landmark/BUILD b/mediapipe/modules/pose_landmark/BUILD index e92da62a80..bb836534d7 100644 --- a/mediapipe/modules/pose_landmark/BUILD +++ b/mediapipe/modules/pose_landmark/BUILD @@ -62,7 +62,7 @@ mediapipe_simple_subgraph( "//mediapipe/calculators/image:image_properties_calculator", "//mediapipe/calculators/tensor:image_to_tensor_calculator", "//mediapipe/calculators/tensor:inference_calculator", - "@mediapipe/graphs/object_detection:desktop_ovms_calculators" + "@//mediapipe/graphs/object_detection:desktop_ovms_calculators" ], )