From 11fe56a9cbcc828d4e262e106bfdabe5ed6803e7 Mon Sep 17 00:00:00 2001 From: Nadezhda Ageeva Date: Tue, 10 Oct 2023 16:13:33 +0400 Subject: [PATCH] [NVIDIA] Move Broadcast, Comparison and Concat tests to API 2.0 (#734) --- .../single_layer_tests/broadcast.cpp | 95 +++++++-------- .../single_layer_tests/comparison.cpp | 107 ++++++++-------- .../single_layer_tests/concat.cpp | 23 ++-- .../single_layer_tests/ov_benchmark.hpp | 114 ++++++++++++++++++ .../ov_unsymmetrical_comparer.hpp | 35 ++++++ 5 files changed, 262 insertions(+), 112 deletions(-) create mode 100644 modules/nvidia_plugin/tests/functional/shared_tests_instances/single_layer_tests/ov_benchmark.hpp create mode 100644 modules/nvidia_plugin/tests/functional/shared_tests_instances/single_layer_tests/ov_unsymmetrical_comparer.hpp diff --git a/modules/nvidia_plugin/tests/functional/shared_tests_instances/single_layer_tests/broadcast.cpp b/modules/nvidia_plugin/tests/functional/shared_tests_instances/single_layer_tests/broadcast.cpp index 5503590b8..b8ff4b2ed 100644 --- a/modules/nvidia_plugin/tests/functional/shared_tests_instances/single_layer_tests/broadcast.cpp +++ b/modules/nvidia_plugin/tests/functional/shared_tests_instances/single_layer_tests/broadcast.cpp @@ -2,17 +2,19 @@ // SPDX-License-Identifier: Apache-2.0 // -#include "single_layer_tests/broadcast.hpp" +#include "single_op_tests/broadcast.hpp" #include #include -using namespace LayerTestsDefinitions; - namespace { -const std::vector inputPrecisions = { - InferenceEngine::Precision::FP16, InferenceEngine::Precision::FP32, InferenceEngine::Precision::I32}; +using namespace ov::test; +using namespace ov::test::utils; + +const std::vector input_precisions = { + ov::element::f16, ov::element::f32, ov::element::i32 +}; // NUMPY MODE @@ -21,9 +23,9 @@ INSTANTIATE_TEST_CASE_P(smoke_TestNumpyBroadcast1, ::testing::Combine(::testing::Values(ov::Shape{2, 3, 6}), // target shape ::testing::Values(ov::AxisSet{}), // not used in numpy mode ::testing::Values(ov::op::BroadcastType::NUMPY), - ::testing::Values(ov::Shape{3, 1}), // input shape - ::testing::ValuesIn(inputPrecisions), - ::testing::Values(ov::test::utils::DEVICE_NVIDIA)), + ::testing::Values(static_shapes_to_test_representation({ov::Shape{3, 1}})), // input shape + ::testing::ValuesIn(input_precisions), + ::testing::Values(DEVICE_NVIDIA)), BroadcastLayerTest::getTestCaseName); INSTANTIATE_TEST_CASE_P(smoke_TestNumpyBroadcast2, @@ -31,9 +33,9 @@ INSTANTIATE_TEST_CASE_P(smoke_TestNumpyBroadcast2, ::testing::Combine(::testing::Values(ov::Shape{1, 4, 4}), // target shape ::testing::Values(ov::AxisSet{}), // not used in numpy mode ::testing::Values(ov::op::BroadcastType::NUMPY), - ::testing::Values(ov::Shape{1, 4, 1}), // input shape - ::testing::ValuesIn(inputPrecisions), - ::testing::Values(ov::test::utils::DEVICE_NVIDIA)), + ::testing::Values(static_shapes_to_test_representation({ov::Shape{1, 4, 1}})), // input shape + ::testing::ValuesIn(input_precisions), + ::testing::Values(DEVICE_NVIDIA)), BroadcastLayerTest::getTestCaseName); INSTANTIATE_TEST_CASE_P(smoke_TestNumpyBroadcast3, @@ -41,9 +43,9 @@ INSTANTIATE_TEST_CASE_P(smoke_TestNumpyBroadcast3, ::testing::Combine(::testing::Values(ov::Shape{3, 1, 4}), // target shape ::testing::Values(ov::AxisSet{}), // not used in numpy mode ::testing::Values(ov::op::BroadcastType::NUMPY), - ::testing::Values(ov::Shape{3, 1, 1}), // input shape - ::testing::ValuesIn(inputPrecisions), - ::testing::Values(ov::test::utils::DEVICE_NVIDIA)), + ::testing::Values(static_shapes_to_test_representation({ov::Shape{3, 1, 1}})), // input shape + ::testing::ValuesIn(input_precisions), + ::testing::Values(DEVICE_NVIDIA)), BroadcastLayerTest::getTestCaseName); INSTANTIATE_TEST_CASE_P(smoke_TestNumpyBroadcast4, @@ -51,9 +53,9 @@ INSTANTIATE_TEST_CASE_P(smoke_TestNumpyBroadcast4, ::testing::Combine(::testing::Values(ov::Shape{2, 3, 3, 3, 3, 3, 3, 3}), // target shape ::testing::Values(ov::AxisSet{}), // not used in numpy mode ::testing::Values(ov::op::BroadcastType::NUMPY), - ::testing::Values(ov::Shape{1, 3, 1, 3, 1, 3, 1}), // input shape - ::testing::ValuesIn(inputPrecisions), - ::testing::Values(ov::test::utils::DEVICE_NVIDIA)), + ::testing::Values(static_shapes_to_test_representation({ov::Shape{1, 3, 1, 3, 1, 3, 1}})), // input shape + ::testing::ValuesIn(input_precisions), + ::testing::Values(DEVICE_NVIDIA)), BroadcastLayerTest::getTestCaseName); // BIDIRECTIONAL MODE @@ -63,9 +65,9 @@ INSTANTIATE_TEST_CASE_P(smoke_TestBidirectionalBroadcast1, ::testing::Combine(::testing::Values(ov::Shape{2, 1, 4}), // target shape ::testing::Values(ov::AxisSet{}), // not used in bidirectional mode ::testing::Values(ov::op::BroadcastType::BIDIRECTIONAL), - ::testing::Values(ov::Shape{4, 1}), // input shape - ::testing::ValuesIn(inputPrecisions), - ::testing::Values(ov::test::utils::DEVICE_NVIDIA)), + ::testing::Values(static_shapes_to_test_representation({ov::Shape{4, 1}})), // input shape + ::testing::ValuesIn(input_precisions), + ::testing::Values(DEVICE_NVIDIA)), BroadcastLayerTest::getTestCaseName); INSTANTIATE_TEST_CASE_P(smoke_TestBidirectionalBroadcast2, @@ -73,9 +75,9 @@ INSTANTIATE_TEST_CASE_P(smoke_TestBidirectionalBroadcast2, ::testing::Combine(::testing::Values(ov::Shape{1, 4, 4}), // target shape ::testing::Values(ov::AxisSet{}), // not used in bidirectional mode ::testing::Values(ov::op::BroadcastType::BIDIRECTIONAL), - ::testing::Values(ov::Shape{1, 4, 1}), // input shape - ::testing::ValuesIn(inputPrecisions), - ::testing::Values(ov::test::utils::DEVICE_NVIDIA)), + ::testing::Values(static_shapes_to_test_representation({ov::Shape{1, 4, 1}})), // input shape + ::testing::ValuesIn(input_precisions), + ::testing::Values(DEVICE_NVIDIA)), BroadcastLayerTest::getTestCaseName); INSTANTIATE_TEST_CASE_P(smoke_TestBidirectionalBroadcas3, @@ -83,9 +85,9 @@ INSTANTIATE_TEST_CASE_P(smoke_TestBidirectionalBroadcas3, ::testing::Combine(::testing::Values(ov::Shape{1, 1, 2, 2}), // target shape ::testing::Values(ov::AxisSet{}), // not used in bidirectional mode ::testing::Values(ov::op::BroadcastType::BIDIRECTIONAL), - ::testing::Values(ov::Shape{4, 1, 1}), // input shape - ::testing::ValuesIn(inputPrecisions), - ::testing::Values(ov::test::utils::DEVICE_NVIDIA)), + ::testing::Values(static_shapes_to_test_representation({ov::Shape{4, 1, 1}})), // input shape + ::testing::ValuesIn(input_precisions), + ::testing::Values(DEVICE_NVIDIA)), BroadcastLayerTest::getTestCaseName); // EXPLICIT MODE @@ -95,9 +97,9 @@ INSTANTIATE_TEST_CASE_P(smoke_TestExplicitBroadcast1, ::testing::Combine(::testing::Values(ov::Shape{2, 3, 1}), // target shape ::testing::Values(ov::AxisSet{1, 2}), // axes ::testing::Values(ov::op::BroadcastType::EXPLICIT), - ::testing::Values(ov::Shape{3, 1}), // input shape - ::testing::ValuesIn(inputPrecisions), - ::testing::Values(ov::test::utils::DEVICE_NVIDIA)), + ::testing::Values(static_shapes_to_test_representation({ov::Shape{3, 1}})), // input shape + ::testing::ValuesIn(input_precisions), + ::testing::Values(DEVICE_NVIDIA)), BroadcastLayerTest::getTestCaseName); INSTANTIATE_TEST_CASE_P(smoke_TestExplicitBroadcast2, @@ -105,9 +107,9 @@ INSTANTIATE_TEST_CASE_P(smoke_TestExplicitBroadcast2, ::testing::Combine(::testing::Values(ov::Shape{5, 3, 7}), // target shape ::testing::Values(ov::AxisSet{0, 2}), // axes ::testing::Values(ov::op::BroadcastType::EXPLICIT), - ::testing::Values(ov::Shape{5, 7}), // input shape - ::testing::ValuesIn(inputPrecisions), - ::testing::Values(ov::test::utils::DEVICE_NVIDIA)), + ::testing::Values(static_shapes_to_test_representation({ov::Shape{5, 7}})), // input shape + ::testing::ValuesIn(input_precisions), + ::testing::Values(DEVICE_NVIDIA)), BroadcastLayerTest::getTestCaseName); INSTANTIATE_TEST_CASE_P(smoke_TestExplicitBroadcast3, @@ -115,16 +117,15 @@ INSTANTIATE_TEST_CASE_P(smoke_TestExplicitBroadcast3, ::testing::Combine(::testing::Values(ov::Shape{4, 4, 3, 7, 6, 6}), // target shape ::testing::Values(ov::AxisSet{1, 3, 5}), // axes ::testing::Values(ov::op::BroadcastType::EXPLICIT), - ::testing::Values(ov::Shape{4, 7, 6}), // input shape - ::testing::ValuesIn(inputPrecisions), - ::testing::Values(ov::test::utils::DEVICE_NVIDIA)), + ::testing::Values(static_shapes_to_test_representation({ov::Shape{4, 7, 6}})), // input shape + ::testing::ValuesIn(input_precisions), + ::testing::Values(DEVICE_NVIDIA)), BroadcastLayerTest::getTestCaseName); // YOLOv5 operators -const std::vector precisionsYOLOv5 = { - InferenceEngine::Precision::FP16, - InferenceEngine::Precision::FP32, +const std::vector precisions_YOLOv5 = { + ov::element::f16, ov::element::f32 }; INSTANTIATE_TEST_CASE_P(yolov5_BroadcastTest1, @@ -132,9 +133,9 @@ INSTANTIATE_TEST_CASE_P(yolov5_BroadcastTest1, ::testing::Combine(::testing::Values(ov::Shape{1, 3, 80, 80, 2}), ::testing::Values(ov::AxisSet{}), // not used in bidirectional mode ::testing::Values(ov::op::BroadcastType::BIDIRECTIONAL), - ::testing::Values(ov::Shape{1, 3, 80, 80, 2}), - ::testing::ValuesIn(precisionsYOLOv5), - ::testing::Values(ov::test::utils::DEVICE_NVIDIA)), + ::testing::Values(static_shapes_to_test_representation({ov::Shape{1, 3, 80, 80, 2}})), + ::testing::ValuesIn(precisions_YOLOv5), + ::testing::Values(DEVICE_NVIDIA)), BroadcastLayerTest::getTestCaseName); INSTANTIATE_TEST_CASE_P(yolov5_BroadcastTest2, @@ -142,9 +143,9 @@ INSTANTIATE_TEST_CASE_P(yolov5_BroadcastTest2, ::testing::Combine(::testing::Values(ov::Shape{1, 3, 40, 40, 2}), ::testing::Values(ov::AxisSet{}), // not used in bidirectional mode ::testing::Values(ov::op::BroadcastType::BIDIRECTIONAL), - ::testing::Values(ov::Shape{1, 3, 40, 40, 2}), - ::testing::ValuesIn(precisionsYOLOv5), - ::testing::Values(ov::test::utils::DEVICE_NVIDIA)), + ::testing::Values(static_shapes_to_test_representation({ov::Shape{1, 3, 40, 40, 2}})), + ::testing::ValuesIn(precisions_YOLOv5), + ::testing::Values(DEVICE_NVIDIA)), BroadcastLayerTest::getTestCaseName); INSTANTIATE_TEST_CASE_P(yolov5_BroadcastTest3, @@ -152,9 +153,9 @@ INSTANTIATE_TEST_CASE_P(yolov5_BroadcastTest3, ::testing::Combine(::testing::Values(ov::Shape{1, 3, 20, 20, 2}), ::testing::Values(ov::AxisSet{}), // not used in bidirectional mode ::testing::Values(ov::op::BroadcastType::BIDIRECTIONAL), - ::testing::Values(ov::Shape{1, 3, 20, 20, 2}), - ::testing::ValuesIn(precisionsYOLOv5), - ::testing::Values(ov::test::utils::DEVICE_NVIDIA)), + ::testing::Values(static_shapes_to_test_representation({ov::Shape{1, 3, 20, 20, 2}})), + ::testing::ValuesIn(precisions_YOLOv5), + ::testing::Values(DEVICE_NVIDIA)), BroadcastLayerTest::getTestCaseName); } // namespace diff --git a/modules/nvidia_plugin/tests/functional/shared_tests_instances/single_layer_tests/comparison.cpp b/modules/nvidia_plugin/tests/functional/shared_tests_instances/single_layer_tests/comparison.cpp index 29fd1370b..39a972fb2 100644 --- a/modules/nvidia_plugin/tests/functional/shared_tests_instances/single_layer_tests/comparison.cpp +++ b/modules/nvidia_plugin/tests/functional/shared_tests_instances/single_layer_tests/comparison.cpp @@ -2,27 +2,23 @@ // SPDX-License-Identifier: Apache-2.0 // -#include "single_layer_tests/comparison.hpp" +#include "shared_test_classes/single_op/comparison.hpp" #include -#include "common_test_utils/test_constants.hpp" #include "cuda_test_constants.hpp" -#include "unsymmetrical_comparer.hpp" - -using namespace LayerTestsDefinitions; -using namespace LayerTestsDefinitions::ComparisonParams; - -namespace LayerTestsDefinitions { +#include "functional_test_utils/skip_tests_config.hpp" +#include "ov_unsymmetrical_comparer.hpp" +namespace { +using namespace ov::test; +using namespace ov::test::utils; +using namespace ngraph::helpers; class UnsymmetricalComparisonLayerTest : public UnsymmetricalComparer {}; -TEST_P(UnsymmetricalComparisonLayerTest, CompareWithRefs) { Run(); } -} // namespace LayerTestsDefinitions +TEST_P(UnsymmetricalComparisonLayerTest, Inference) { run(); } -namespace { - -std::map, std::vector>> smokeShapes = { +std::map> smoke_shapes = { {{1}, {{1}, {17}, {1, 1}, {2, 18}, {1, 1, 2}, {2, 2, 3}, {1, 1, 2, 3}}}, {{5}, {{1}, {1, 1}, {2, 5}, {1, 1, 1}, {2, 2, 5}}}, {{2, 200}, {{1}, {200}, {1, 200}, {2, 200}, {2, 2, 200}}}, @@ -30,79 +26,86 @@ std::map, std::vector>> smokeShapes = { {{2, 17, 3, 4}, {{4}, {1, 3, 4}, {2, 1, 3, 4}}}, {{2, 1, 1, 3, 1}, {{1}, {1, 3, 4}, {2, 1, 3, 4}, {1, 1, 1, 1, 1}}}, }; +auto converter = [] (const std::vector>& shapes) { + std::vector> result; + for (const auto& shape : shapes) { + result.push_back({shape.first, shape.second}); + } + return result; +}; +auto input_smoke_shapes_pair_vector = ov::test::utils::combineParams(smoke_shapes); +auto input_smoke_shapes_static = converter(input_smoke_shapes_pair_vector); -std::map, std::vector>> shapes = { +std::map> shapes = { {{1}, {{256}}}, }; +auto input_shapes_pair_vector = ov::test::utils::combineParams(shapes); +auto input_shapes_static = converter(input_shapes_pair_vector); -std::vector inputsPrecisions = { - InferenceEngine::Precision::FP16, - InferenceEngine::Precision::FP32, +std::vector model_type = { + ov::element::f32, + ov::element::f16, }; -std::vector comparisonOpTypes = { - ngraph::helpers::ComparisonTypes::EQUAL, - ngraph::helpers::ComparisonTypes::NOT_EQUAL, - ngraph::helpers::ComparisonTypes::GREATER, - ngraph::helpers::ComparisonTypes::GREATER_EQUAL, - ngraph::helpers::ComparisonTypes::LESS, - ngraph::helpers::ComparisonTypes::LESS_EQUAL, +std::vector comparison_op_types = { + ComparisonTypes::EQUAL, + ComparisonTypes::NOT_EQUAL, + ComparisonTypes::GREATER, + ComparisonTypes::GREATER_EQUAL, + ComparisonTypes::LESS, + ComparisonTypes::LESS_EQUAL, }; -std::vector secondInputTypes = { - ngraph::helpers::InputLayerType::CONSTANT, - ngraph::helpers::InputLayerType::PARAMETER, +std::vector second_input_types = { + InputLayerType::CONSTANT, + InputLayerType::PARAMETER, }; std::map additional_config = {}; -const auto smokeComparisonTestParams = - ::testing::Combine(::testing::ValuesIn(ov::test::utils::combineParams(smokeShapes)), - ::testing::ValuesIn(inputsPrecisions), - ::testing::ValuesIn(comparisonOpTypes), - ::testing::ValuesIn(secondInputTypes), - ::testing::Values(InferenceEngine::Precision::UNSPECIFIED), - ::testing::Values(InferenceEngine::Precision::UNSPECIFIED), - ::testing::Values(ov::test::utils::DEVICE_NVIDIA), +const auto smoke_comparison_test_params = + ::testing::Combine(::testing::ValuesIn(ov::test::static_shapes_to_test_representation(input_smoke_shapes_static)), + ::testing::ValuesIn(comparison_op_types), + ::testing::ValuesIn(second_input_types), + ::testing::ValuesIn(model_type), + ::testing::Values(DEVICE_NVIDIA), ::testing::Values(additional_config)); -const auto comparisonTestParams = ::testing::Combine(::testing::ValuesIn(ov::test::utils::combineParams(shapes)), - ::testing::ValuesIn(inputsPrecisions), - ::testing::ValuesIn(comparisonOpTypes), - ::testing::ValuesIn(secondInputTypes), - ::testing::Values(InferenceEngine::Precision::UNSPECIFIED), - ::testing::Values(InferenceEngine::Precision::UNSPECIFIED), - ::testing::Values(ov::test::utils::DEVICE_NVIDIA), - ::testing::Values(additional_config)); +const auto comparison_test_params = ::testing::Combine(::testing::ValuesIn(ov::test::static_shapes_to_test_representation(input_shapes_static)), + ::testing::ValuesIn(comparison_op_types), + ::testing::ValuesIn(second_input_types), + ::testing::ValuesIn(model_type), + ::testing::Values(DEVICE_NVIDIA), + ::testing::Values(additional_config)); INSTANTIATE_TEST_CASE_P(smoke_ComparisonCompareWithRefs, UnsymmetricalComparisonLayerTest, - smokeComparisonTestParams, + smoke_comparison_test_params, ComparisonLayerTest::getTestCaseName); INSTANTIATE_TEST_CASE_P(ComparisonCompareWithRefs, UnsymmetricalComparisonLayerTest, - comparisonTestParams, + comparison_test_params, ComparisonLayerTest::getTestCaseName); +} // namespace + // ------------- Benchmark ------------- -#include "benchmark.hpp" +#include "ov_benchmark.hpp" -namespace LayerTestsDefinitions { +namespace { namespace benchmark { struct ComparisonBenchmarkTest : BenchmarkLayerTest {}; TEST_P(ComparisonBenchmarkTest, DISABLED_benchmark) { SKIP_IF_CURRENT_TEST_IS_DISABLED() - Run("Comparison", std::chrono::milliseconds(2000), 100); + run("Comparison", std::chrono::milliseconds(2000), 100); } INSTANTIATE_TEST_CASE_P(smoke_ComparisonCompareWithRefs, ComparisonBenchmarkTest, - smokeComparisonTestParams, + smoke_comparison_test_params, ComparisonLayerTest::getTestCaseName); } // namespace benchmark - -} // namespace LayerTestsDefinitions -} // namespace +} // namespace \ No newline at end of file diff --git a/modules/nvidia_plugin/tests/functional/shared_tests_instances/single_layer_tests/concat.cpp b/modules/nvidia_plugin/tests/functional/shared_tests_instances/single_layer_tests/concat.cpp index e6568ce69..7b708753f 100644 --- a/modules/nvidia_plugin/tests/functional/shared_tests_instances/single_layer_tests/concat.cpp +++ b/modules/nvidia_plugin/tests/functional/shared_tests_instances/single_layer_tests/concat.cpp @@ -2,38 +2,35 @@ // SPDX-License-Identifier: Apache-2.0 // -#include "single_layer_tests/concat.hpp" +#include "single_op_tests/concat.hpp" #include #include "common_test_utils/test_constants.hpp" #include "cuda_test_constants.hpp" -using namespace LayerTestsDefinitions; - namespace { +using namespace ov::test; +using namespace ov::test::utils; + std::vector axes = {-3, -2, -1, 0, 1, 2, 3}; -std::vector>> inShapes = { +std::vector> shapes_static = { {{10, 10, 10, 10}}, {{10, 10, 10, 10}, {10, 10, 10, 10}}, {{10, 10, 10, 10}, {10, 10, 10, 10}, {10, 10, 10, 10}}, {{10, 10, 10, 10}, {10, 10, 10, 10}, {10, 10, 10, 10}, {10, 10, 10, 10}}, {{10, 10, 10, 10}, {10, 10, 10, 10}, {10, 10, 10, 10}, {10, 10, 10, 10}, {10, 10, 10, 10}}}; -std::vector netPrecisions = {InferenceEngine::Precision::FP32, - InferenceEngine::Precision::FP16}; +std::vector model_precisions = {ov::element::f32, + ov::element::f16}; INSTANTIATE_TEST_CASE_P(smoke_NoReshape, ConcatLayerTest, ::testing::Combine(::testing::ValuesIn(axes), - ::testing::ValuesIn(inShapes), - ::testing::ValuesIn(netPrecisions), - ::testing::Values(InferenceEngine::Precision::UNSPECIFIED), - ::testing::Values(InferenceEngine::Precision::UNSPECIFIED), - ::testing::Values(InferenceEngine::Layout::ANY), - ::testing::Values(InferenceEngine::Layout::ANY), - ::testing::Values(ov::test::utils::DEVICE_NVIDIA)), + ::testing::ValuesIn(static_shapes_to_test_representation(shapes_static)), + ::testing::ValuesIn(model_precisions), + ::testing::Values(DEVICE_NVIDIA)), ConcatLayerTest::getTestCaseName); } // namespace diff --git a/modules/nvidia_plugin/tests/functional/shared_tests_instances/single_layer_tests/ov_benchmark.hpp b/modules/nvidia_plugin/tests/functional/shared_tests_instances/single_layer_tests/ov_benchmark.hpp new file mode 100644 index 000000000..8b5518562 --- /dev/null +++ b/modules/nvidia_plugin/tests/functional/shared_tests_instances/single_layer_tests/ov_benchmark.hpp @@ -0,0 +1,114 @@ +// Copyright (C) 2021-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include +#include + +#include "shared_test_classes/base/ov_subgraph.hpp" +#include "openvino/runtime/properties.hpp" + +namespace ov { +namespace test { + +template +class BenchmarkLayerTest : public BaseLayerTest { + static_assert(std::is_base_of::value, + "BaseLayerTest should inherit from ov::test::SubgraphBaseTest"); + +public: + void run(const std::initializer_list& names, + const std::chrono::milliseconds warmupTime = std::chrono::milliseconds(2000), + const int numAttempts = 100) { + bench_names_ = names; + warmup_time_ = warmupTime; + num_attempts_ = numAttempts; + SubgraphBaseTest::configuration = {ov::enable_profiling(true)}; + SubgraphBaseTest::run(); + } + + void run(const std::string& name, + const std::chrono::milliseconds warmupTime = std::chrono::milliseconds(2000), + const int numAttempts = 100) { + if (!name.empty()) { + run({name}, warmupTime, numAttempts); + } else { + run({}, warmupTime, numAttempts); + } + } + + void validate() override { + // NOTE: Validation is ignored because we are interested in benchmarks results + } + +protected: + void infer() override { + // Operation names search + std::map results_us{}; + SubgraphBaseTest::infer(); + const auto& perf_results = SubgraphBaseTest::inferRequest.get_profiling_info(); + for (const auto& name : bench_names_) { + bool found = false; + for (const auto& result : perf_results) { + const auto& res_name = result.node_name; + const bool should_add = + !name.empty() && res_name.find(name) != std::string::npos && res_name.find('_') != std::string::npos; + // Adding operations with numbers for the case there are several operations of the same type + if (should_add) { + found = true; + results_us.emplace(std::make_pair(res_name, std::chrono::microseconds::zero())); + } + } + if (!found) { + std::cout << "WARNING! Performance count for \"" << name << "\" wasn't found!\n"; + } + } + // If no operations were found adding the time of all operations except Parameter and Result + if (results_us.empty()) { + for (const auto& result : perf_results) { + const auto& res_name = result.node_name; + const bool should_add = (res_name.find("Parameter") == std::string::npos) && + (res_name.find("Result") == std::string::npos) && + (res_name.find('_') != std::string::npos); + if (should_add) { + results_us.emplace(std::make_pair(res_name, std::chrono::microseconds::zero())); + } + } + } + // Warmup + auto warmCur = std::chrono::steady_clock::now(); + const auto warmEnd = warmCur + warmup_time_; + while (warmCur < warmEnd) { + SubgraphBaseTest::infer(); + warmCur = std::chrono::steady_clock::now(); + } + // Benchmark + for (int i = 0; i < num_attempts_; ++i) { + SubgraphBaseTest::infer(); + const auto& perf_results = SubgraphBaseTest::inferRequest.get_profiling_info(); + for (auto& [name, time] : results_us) { + auto it = std::find_if(perf_results.begin(), perf_results.end(), [&](const ::ov::ProfilingInfo& info) { return info.node_name == name; }); + OPENVINO_ASSERT(it != perf_results.end()); + time += it->real_time; + } + } + + std::chrono::microseconds total_us = std::chrono::microseconds::zero(); + for (auto& [name, time] : results_us) { + time /= num_attempts_; + total_us += time; + std::cout << std::fixed << std::setfill('0') << name << ": " << time.count() << " us\n"; + } + std::cout << std::fixed << std::setfill('0') << "Total time: " << total_us.count() << " us\n"; + } + +private: + std::vector bench_names_; + std::chrono::milliseconds warmup_time_; + int num_attempts_; +}; + +} // namespace test +} // namespace ov diff --git a/modules/nvidia_plugin/tests/functional/shared_tests_instances/single_layer_tests/ov_unsymmetrical_comparer.hpp b/modules/nvidia_plugin/tests/functional/shared_tests_instances/single_layer_tests/ov_unsymmetrical_comparer.hpp new file mode 100644 index 000000000..6ac83dc81 --- /dev/null +++ b/modules/nvidia_plugin/tests/functional/shared_tests_instances/single_layer_tests/ov_unsymmetrical_comparer.hpp @@ -0,0 +1,35 @@ +// Copyright (C) 2021-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "common_test_utils/test_common.hpp" +#include "common_test_utils/ov_tensor_utils.hpp" +#include "openvino/core/model.hpp" + +namespace ov { +namespace test { + +template +class UnsymmetricalComparer : public BaseLayerTest, virtual public ov::test::SubgraphBaseTest { +protected: + ov::Tensor generate_input(const ov::element::Type element_type, const ov::Shape& shape, int seed) const { + return ov::test::utils::create_and_fill_tensor(element_type, shape, 10, 0, 1, seed); + }; + + void generate_inputs(const std::vector& target_input_static_shapes) override { + inputs.clear(); + const auto& func_inputs = function->inputs(); + const int base_seed = 1; + + for (int i = 0; i < func_inputs.size(); ++i) { + + const auto& param = func_inputs[i]; + auto tensor = generate_input(param.get_element_type(), target_input_static_shapes[i], base_seed + i); + inputs.insert({param.get_node_shared_ptr(), tensor}); + } + } +}; +} // namespace test +} // namespace ov