From e10f512d52b3a3a7527cb4347bdf16644c6726af Mon Sep 17 00:00:00 2001 From: Pawel Raasz Date: Wed, 9 Oct 2024 13:34:28 +0000 Subject: [PATCH 01/21] Split configuration for device and others in compile model --- src/inference/src/cpp/core.cpp | 10 ++- src/inference/src/dev/core_impl.cpp | 61 ++++++++++++++++--- src/inference/src/dev/core_impl.hpp | 18 ++++++ .../ov_infer_request/properties_tests.hpp | 13 ++++ 4 files changed, 92 insertions(+), 10 deletions(-) diff --git a/src/inference/src/cpp/core.cpp b/src/inference/src/cpp/core.cpp index 2d6c204757bcf6..eb2017bbb8b804 100644 --- a/src/inference/src/cpp/core.cpp +++ b/src/inference/src/cpp/core.cpp @@ -102,6 +102,8 @@ CompiledModel Core::compile_model(const std::shared_ptr& model, const std::string& device_name, const AnyMap& config) { OV_CORE_CALL_STATEMENT({ + const auto device_not_supported = _impl->get_device_unsupported_properties(device_name, config); + set_property(device_not_supported); auto exec = _impl->compile_model(model, device_name, config); return {exec._ptr, exec._so}; }); @@ -119,6 +121,8 @@ CompiledModel Core::compile_model(const std::wstring& model_path, const AnyMap& CompiledModel Core::compile_model(const std::string& model_path, const std::string& device_name, const AnyMap& config) { OV_CORE_CALL_STATEMENT({ + const auto device_not_supported = _impl->get_device_unsupported_properties(device_name, config); + set_property(device_not_supported); auto exec = _impl->compile_model(model_path, device_name, config); return {exec._ptr, exec._so}; }); @@ -137,7 +141,9 @@ CompiledModel Core::compile_model(const std::string& model, const std::string& device_name, const AnyMap& config) { OV_CORE_CALL_STATEMENT({ - auto exec = _impl->compile_model(model, weights, device_name, config); + const auto device_not_supported = _impl->get_device_unsupported_properties(device_name, config); + set_property(device_not_supported); + auto exec = _impl->compile_model(model, device_name, config); return {exec._ptr, exec._so}; }); } @@ -146,6 +152,8 @@ CompiledModel Core::compile_model(const std::shared_ptr& model, const RemoteContext& context, const AnyMap& config) { OV_CORE_CALL_STATEMENT({ + const auto device_not_supported = _impl->get_device_unsupported_properties(context.get_device_name(), config); + set_property(device_not_supported); auto exec = _impl->compile_model(model, ov::SoPtr{context._impl, context._so}, config); return {exec._ptr, exec._so}; }); diff --git a/src/inference/src/dev/core_impl.cpp b/src/inference/src/dev/core_impl.cpp index 237c246ab38bdc..c0ffedc36541d1 100644 --- a/src/inference/src/dev/core_impl.cpp +++ b/src/inference/src/dev/core_impl.cpp @@ -204,6 +204,28 @@ void clean_batch_properties(const std::string& deviceName, ov::AnyMap& config, c } } } + +/** + * @brief Splits input configuration on two parts: supported and others + * + * @param config Configuration with properties to be splitted. + * @param supported List of supported configuration properties. + * @return Pair of configs where first is supported properties, second are others. + */ +std::pair split_config(const ov::AnyMap& config, const ov::AnyMap& supported) { + auto split_config = std::make_pair(ov::AnyMap{}, config); + auto& plugin_config = split_config.first; + auto& others_config = split_config.second; + + for (const auto& supported_property : supported) { + auto config_property = config.find(supported_property.first); + if (config_property != config.end()) { + plugin_config.insert(plugin_config.end(), *config_property); + others_config.erase(supported_property.first); + } + } + return split_config; +} } // namespace bool ov::is_config_applicable(const std::string& user_device_name, const std::string& subprop_device_name) { @@ -748,12 +770,12 @@ ov::SoPtr ov::CoreImpl::compile_model(const std::shared_ptr< res = load_model_from_cache(cacheContent, plugin, parsed._config, ov::SoPtr{}, [&]() { return compile_model_and_cache(plugin, model, - parsed._config, + get_device_properties(device_name, parsed._config), ov::SoPtr{}, cacheContent); }); } else { - res = plugin.compile_model(model, parsed._config); + res = plugin.compile_model(model, get_device_properties(device_name, parsed._config)); } return res; } @@ -779,10 +801,14 @@ ov::SoPtr ov::CoreImpl::compile_model(const std::shared_ptr< cacheContent.blobId = ov::ModelCache::compute_hash(model, create_compile_config(plugin, parsed._config)); std::unique_ptr lock = cacheGuard.get_hash_lock(cacheContent.blobId); res = load_model_from_cache(cacheContent, plugin, parsed._config, context, [&]() { - return compile_model_and_cache(plugin, model, parsed._config, context, cacheContent); + return compile_model_and_cache(plugin, + model, + get_device_properties(deviceName, parsed._config), + context, + cacheContent); }); } else { - res = plugin.compile_model(model, context, parsed._config); + res = plugin.compile_model(model, context, get_device_properties(deviceName, parsed._config)); } return res; } @@ -797,7 +823,6 @@ ov::SoPtr ov::CoreImpl::compile_model(const std::string& mod ov::SoPtr compiled_model; auto cacheManager = coreConfig.get_cache_config_for_device(plugin, parsed._config)._cacheManager; - if (cacheManager && device_supports_model_caching(plugin) && !is_proxy_device(plugin)) { // Skip caching for proxy plugin. HW plugin will load network from the cache CacheContent cacheContent{cacheManager, model_path}; @@ -806,10 +831,14 @@ ov::SoPtr ov::CoreImpl::compile_model(const std::string& mod compiled_model = load_model_from_cache(cacheContent, plugin, parsed._config, ov::SoPtr{}, [&]() { auto model = read_model(model_path, std::string{}); - return compile_model_and_cache(plugin, model, parsed._config, {}, cacheContent); + return compile_model_and_cache(plugin, + model, + get_device_properties(device_name, parsed._config), + {}, + cacheContent); }); } else { - compiled_model = plugin.compile_model(model_path, parsed._config); + compiled_model = plugin.compile_model(model_path, get_device_properties(device_name, parsed._config)); } return compiled_model; } @@ -836,13 +865,13 @@ ov::SoPtr ov::CoreImpl::compile_model(const std::string& mod auto model = read_model(model_str, weights); return compile_model_and_cache(plugin, model, - parsed._config, + get_device_properties(device_name, parsed._config), ov::SoPtr{}, cacheContent); }); } else { auto model = read_model(model_str, weights); - compiled_model = plugin.compile_model(model, parsed._config); + compiled_model = plugin.compile_model(model, get_device_properties(device_name, parsed._config)); } return compiled_model; } @@ -1655,3 +1684,17 @@ std::map ov::CoreImpl::get_versions(const std::string& return versions; } + +ov::AnyMap ov::CoreImpl::get_device_properties(const std::string& device_name, const AnyMap& config) const { + const auto parsed = parseDeviceNameIntoConfig(device_name, config); + return is_virtual_device(parsed._deviceName) + ? parsed._config + : split_config(parsed._config, get_supported_property(parsed._deviceName, parsed._config, false)).first; +} + +ov::AnyMap ov::CoreImpl::get_device_unsupported_properties(const std::string& device_name, const AnyMap& config) const { + const auto parsed = parseDeviceNameIntoConfig(device_name, config); + return is_virtual_device(parsed._deviceName) + ? AnyMap{} + : split_config(parsed._config, get_supported_property(parsed._deviceName, parsed._config, false)).second; +} diff --git a/src/inference/src/dev/core_impl.hpp b/src/inference/src/dev/core_impl.hpp index 40f2a15bb725e0..297f703c5cf926 100644 --- a/src/inference/src/dev/core_impl.hpp +++ b/src/inference/src/dev/core_impl.hpp @@ -317,6 +317,24 @@ class CoreImpl : public ov::ICore, public std::enable_shared_from_thisset_property(target_device, configuration)); + } + // Compile model to target plugins + execNet = core->compile_model(function, target_device, configuration); + OV_ASSERT_NO_THROW(execNet.create_infer_request()); +} } // namespace behavior } // namespace test } // namespace ov From 32750d60b75242d42da929c5877d5eac4cc2811d Mon Sep 17 00:00:00 2001 From: Pawel Raasz Date: Wed, 9 Oct 2024 13:38:40 +0000 Subject: [PATCH 02/21] Fix call compile model with weights --- src/inference/src/cpp/core.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/inference/src/cpp/core.cpp b/src/inference/src/cpp/core.cpp index eb2017bbb8b804..48feb187454660 100644 --- a/src/inference/src/cpp/core.cpp +++ b/src/inference/src/cpp/core.cpp @@ -143,7 +143,7 @@ CompiledModel Core::compile_model(const std::string& model, OV_CORE_CALL_STATEMENT({ const auto device_not_supported = _impl->get_device_unsupported_properties(device_name, config); set_property(device_not_supported); - auto exec = _impl->compile_model(model, device_name, config); + auto exec = _impl->compile_model(model, weights, device_name, config); return {exec._ptr, exec._so}; }); } From cdc565552456b660ca0d8362a750d193bffe8cea Mon Sep 17 00:00:00 2001 From: Pawel Raasz Date: Sun, 13 Oct 2024 19:07:34 +0000 Subject: [PATCH 03/21] Remove modify core state in compile_model --- src/inference/src/cpp/core.cpp | 8 -- src/inference/src/dev/core_impl.cpp | 121 +++++++++--------- src/inference/src/dev/core_impl.hpp | 28 ++-- .../functional/behavior/caching_test.cpp | 2 +- src/plugins/intel_cpu/src/plugin.cpp | 3 +- .../compiled_model/compiled_model_base.hpp | 7 +- 6 files changed, 71 insertions(+), 98 deletions(-) diff --git a/src/inference/src/cpp/core.cpp b/src/inference/src/cpp/core.cpp index 48feb187454660..2d6c204757bcf6 100644 --- a/src/inference/src/cpp/core.cpp +++ b/src/inference/src/cpp/core.cpp @@ -102,8 +102,6 @@ CompiledModel Core::compile_model(const std::shared_ptr& model, const std::string& device_name, const AnyMap& config) { OV_CORE_CALL_STATEMENT({ - const auto device_not_supported = _impl->get_device_unsupported_properties(device_name, config); - set_property(device_not_supported); auto exec = _impl->compile_model(model, device_name, config); return {exec._ptr, exec._so}; }); @@ -121,8 +119,6 @@ CompiledModel Core::compile_model(const std::wstring& model_path, const AnyMap& CompiledModel Core::compile_model(const std::string& model_path, const std::string& device_name, const AnyMap& config) { OV_CORE_CALL_STATEMENT({ - const auto device_not_supported = _impl->get_device_unsupported_properties(device_name, config); - set_property(device_not_supported); auto exec = _impl->compile_model(model_path, device_name, config); return {exec._ptr, exec._so}; }); @@ -141,8 +137,6 @@ CompiledModel Core::compile_model(const std::string& model, const std::string& device_name, const AnyMap& config) { OV_CORE_CALL_STATEMENT({ - const auto device_not_supported = _impl->get_device_unsupported_properties(device_name, config); - set_property(device_not_supported); auto exec = _impl->compile_model(model, weights, device_name, config); return {exec._ptr, exec._so}; }); @@ -152,8 +146,6 @@ CompiledModel Core::compile_model(const std::shared_ptr& model, const RemoteContext& context, const AnyMap& config) { OV_CORE_CALL_STATEMENT({ - const auto device_not_supported = _impl->get_device_unsupported_properties(context.get_device_name(), config); - set_property(device_not_supported); auto exec = _impl->compile_model(model, ov::SoPtr{context._impl, context._so}, config); return {exec._ptr, exec._so}; }); diff --git a/src/inference/src/dev/core_impl.cpp b/src/inference/src/dev/core_impl.cpp index c0ffedc36541d1..741cf4bdbf0193 100644 --- a/src/inference/src/dev/core_impl.cpp +++ b/src/inference/src/dev/core_impl.cpp @@ -205,27 +205,7 @@ void clean_batch_properties(const std::string& deviceName, ov::AnyMap& config, c } } -/** - * @brief Splits input configuration on two parts: supported and others - * - * @param config Configuration with properties to be splitted. - * @param supported List of supported configuration properties. - * @return Pair of configs where first is supported properties, second are others. - */ -std::pair split_config(const ov::AnyMap& config, const ov::AnyMap& supported) { - auto split_config = std::make_pair(ov::AnyMap{}, config); - auto& plugin_config = split_config.first; - auto& others_config = split_config.second; - - for (const auto& supported_property : supported) { - auto config_property = config.find(supported_property.first); - if (config_property != config.end()) { - plugin_config.insert(plugin_config.end(), *config_property); - others_config.erase(supported_property.first); - } - } - return split_config; -} +constexpr bool rw_only = true; } // namespace bool ov::is_config_applicable(const std::string& user_device_name, const std::string& subprop_device_name) { @@ -565,8 +545,8 @@ void ov::CoreImpl::register_plugins_in_registry(const std::string& xml_config_fi } } -ov::Plugin ov::CoreImpl::get_plugin(const std::string& pluginName) const { - OV_ITT_SCOPE(FIRST_INFERENCE, ov::itt::domains::LoadTime, "CoreImpl::get_plugin"); +ov::Plugin ov::CoreImpl::get_plugin(const std::string& pluginName, bool on_create_filter_config) const { + OV_ITT_SCOPE(FIRST_INFERENCE, ov::itt::domains::LoadTime, "CoreImpl::get_plugin::supported_config"); auto deviceName = pluginName; if (deviceName == ov::DEFAULT_DEVICE_NAME) @@ -711,13 +691,17 @@ ov::Plugin ov::CoreImpl::get_plugin(const std::string& pluginName) const { ov::DeviceIDParser parser(pluginDesc.first); if (pluginDesc.first.find(deviceName) != std::string::npos && !parser.get_device_id().empty()) { pluginDesc.second.defaultConfig[deviceKey] = parser.get_device_id(); - plugin.set_property(pluginDesc.second.defaultConfig); + plugin.set_property( + on_create_filter_config + ? get_hw_plugin_properties_or_forward(plugin, pluginDesc.second.defaultConfig) + : pluginDesc.second.defaultConfig); } } } - // set global device-id independent settings to plugin - plugin.set_property(desc.defaultConfig); + plugin.set_property(on_create_filter_config + ? get_hw_plugin_properties_or_forward(plugin, desc.defaultConfig) + : desc.defaultConfig); }); } @@ -749,6 +733,12 @@ ov::Plugin ov::CoreImpl::get_plugin(const std::string& pluginName) const { } } +ov::Plugin ov::CoreImpl::get_plugin(const std::string& pluginName) const { + OV_ITT_SCOPE(FIRST_INFERENCE, ov::itt::domains::LoadTime, "CoreImpl::get_plugin"); + constexpr auto on_plugin_create_filter_config = false; + return get_plugin(pluginName, on_plugin_create_filter_config); +} + ov::SoPtr ov::CoreImpl::compile_model(const std::shared_ptr& model_, const std::string& device_name, const ov::AnyMap& config) const { @@ -759,7 +749,7 @@ ov::SoPtr ov::CoreImpl::compile_model(const std::shared_ptr< auto model = apply_auto_batching(model_, deviceName, config_with_batch); auto parsed = parseDeviceNameIntoConfig(deviceName, config_with_batch, is_proxy_device(device_name)); - auto plugin = get_plugin(parsed._deviceName); + auto plugin = get_plugin(parsed._deviceName, !is_virtual_device(parsed._deviceName)); ov::SoPtr res; auto cacheManager = coreConfig.get_cache_config_for_device(plugin, parsed._config)._cacheManager; // Skip caching for proxy plugin. HW plugin will load network from the cache @@ -770,12 +760,12 @@ ov::SoPtr ov::CoreImpl::compile_model(const std::shared_ptr< res = load_model_from_cache(cacheContent, plugin, parsed._config, ov::SoPtr{}, [&]() { return compile_model_and_cache(plugin, model, - get_device_properties(device_name, parsed._config), + get_hw_plugin_properties_or_forward(plugin, parsed._config), ov::SoPtr{}, cacheContent); }); } else { - res = plugin.compile_model(model, get_device_properties(device_name, parsed._config)); + res = plugin.compile_model(model, get_hw_plugin_properties_or_forward(plugin, parsed._config)); } return res; } @@ -792,7 +782,7 @@ ov::SoPtr ov::CoreImpl::compile_model(const std::shared_ptr< auto model = apply_auto_batching(model_, deviceName, config_with_batch); auto parsed = parseDeviceNameIntoConfig(deviceName, config_with_batch, is_proxy_device(deviceName)); - auto plugin = get_plugin(parsed._deviceName); + auto plugin = get_plugin(parsed._deviceName, !is_virtual_device(parsed._deviceName)); ov::SoPtr res; auto cacheManager = coreConfig.get_cache_config_for_device(plugin, parsed._config)._cacheManager; // Skip caching for proxy plugin. HW plugin will load network from the cache @@ -803,12 +793,12 @@ ov::SoPtr ov::CoreImpl::compile_model(const std::shared_ptr< res = load_model_from_cache(cacheContent, plugin, parsed._config, context, [&]() { return compile_model_and_cache(plugin, model, - get_device_properties(deviceName, parsed._config), + get_hw_plugin_properties_or_forward(plugin, parsed._config), context, cacheContent); }); } else { - res = plugin.compile_model(model, context, get_device_properties(deviceName, parsed._config)); + res = plugin.compile_model(model, context, get_hw_plugin_properties_or_forward(plugin, parsed._config)); } return res; } @@ -819,7 +809,7 @@ ov::SoPtr ov::CoreImpl::compile_model(const std::string& mod OV_ITT_SCOPE(FIRST_INFERENCE, ov::itt::domains::LoadTime, "Core::compile_model::Path"); auto parsed = parseDeviceNameIntoConfig(device_name, config); // in case of compile_model(file_name), we need to clear-up core-level properties - auto plugin = get_plugin(parsed._deviceName); + auto plugin = get_plugin(parsed._deviceName, !is_virtual_device(parsed._deviceName)); ov::SoPtr compiled_model; auto cacheManager = coreConfig.get_cache_config_for_device(plugin, parsed._config)._cacheManager; @@ -833,12 +823,12 @@ ov::SoPtr ov::CoreImpl::compile_model(const std::string& mod auto model = read_model(model_path, std::string{}); return compile_model_and_cache(plugin, model, - get_device_properties(device_name, parsed._config), + get_hw_plugin_properties_or_forward(plugin, parsed._config), {}, cacheContent); }); } else { - compiled_model = plugin.compile_model(model_path, get_device_properties(device_name, parsed._config)); + compiled_model = plugin.compile_model(model_path, get_hw_plugin_properties_or_forward(plugin, parsed._config)); } return compiled_model; } @@ -850,7 +840,7 @@ ov::SoPtr ov::CoreImpl::compile_model(const std::string& mod OV_ITT_SCOPED_TASK(ov::itt::domains::OV, "Core::compile_model::from_memory"); auto parsed = parseDeviceNameIntoConfig(device_name, config); // in case of compile_model(file_name), we need to clear-up core-level properties - auto plugin = get_plugin(parsed._deviceName); + auto plugin = get_plugin(parsed._deviceName, !is_virtual_device(parsed._deviceName)); ov::SoPtr compiled_model; auto cacheManager = coreConfig.get_cache_config_for_device(plugin, parsed._config)._cacheManager; @@ -865,13 +855,13 @@ ov::SoPtr ov::CoreImpl::compile_model(const std::string& mod auto model = read_model(model_str, weights); return compile_model_and_cache(plugin, model, - get_device_properties(device_name, parsed._config), + get_hw_plugin_properties_or_forward(plugin, parsed._config), ov::SoPtr{}, cacheContent); }); } else { auto model = read_model(model_str, weights); - compiled_model = plugin.compile_model(model, get_device_properties(device_name, parsed._config)); + compiled_model = plugin.compile_model(model, get_hw_plugin_properties_or_forward(plugin, parsed._config)); } return compiled_model; } @@ -962,9 +952,11 @@ ov::SoPtr ov::CoreImpl::create_context(const std::string& de return get_plugin(parsed._deviceName).create_context(parsed._config); } -ov::AnyMap ov::CoreImpl::get_supported_property(const std::string& full_device_name, - const ov::AnyMap& user_properties, - const bool keep_core_property) const { +ov::AnyMap ov::CoreImpl::get_supported_property(const Plugin& plugin, + const ov::AnyMap& config, + bool keep_core, + bool rw_only) const { + const auto& full_device_name = plugin.get_name(); if (is_virtual_device(full_device_name)) { // Considerations: // 1. in case of virtual devices all the magic will happen on the level when @@ -976,12 +968,11 @@ ov::AnyMap ov::CoreImpl::get_supported_property(const std::string& full_device_n // ov::device::priority cannot be shared, because it's specific for current virtual // plugin. So, we need to remove ov::device::priorities from the list, because it's // supposed to be set for current virtual plugin and cannot be propagated down - ov::AnyMap return_properties = user_properties; + auto return_properties = config; auto device_priorities_it = return_properties.find(ov::device::priorities.name()); if (device_priorities_it != return_properties.end()) { return_properties.erase(device_priorities_it); } - return return_properties; } @@ -993,21 +984,23 @@ ov::AnyMap ov::CoreImpl::get_supported_property(const std::string& full_device_n ov::hint::allow_auto_batching.name(), }; - const auto flattened = ov::parseDeviceNameIntoConfig(full_device_name, user_properties, true); - const std::string& device_name = flattened._deviceName; + const auto flattened = ov::parseDeviceNameIntoConfig(full_device_name, config, true); const auto& flattened_config = flattened._config; // virtual plugins should bypass core-level properties to HW plugins // so, we need to report them as supported std::vector supported_config_keys; - if (keep_core_property) { + if (keep_core) { supported_config_keys = core_level_properties; } + const auto may_add_supported = rw_only ? [](const PropertyName& p){return p.is_mutable();} : [](const + PropertyName&){return true;}; + // try to search against OV API 2.0' mutable supported_properties try { - for (auto&& property : ICore::get_property(device_name, ov::supported_properties, {})) { - if (property.is_mutable()) { + for (auto&& property : plugin.get_property(ov::supported_properties)) { + if (may_add_supported(property)) { supported_config_keys.emplace_back(std::move(property)); } } @@ -1016,8 +1009,8 @@ ov::AnyMap ov::CoreImpl::get_supported_property(const std::string& full_device_n // try to search against internal supported_properties try { - for (auto&& property : ICore::get_property(device_name, ov::internal::supported_properties, {})) { - if (property.is_mutable()) { + for (auto&& property : plugin.get_property(ov::internal::supported_properties)) { + if (may_add_supported(property)) { supported_config_keys.emplace_back(std::move(property)); } } @@ -1035,6 +1028,20 @@ ov::AnyMap ov::CoreImpl::get_supported_property(const std::string& full_device_n return supported_config; } +ov::AnyMap ov::CoreImpl::get_supported_property(const std::string& full_device_name, + const ov::AnyMap& user_properties, + const bool keep_core_property) const { + const auto parsed = ov::parseDeviceNameIntoConfig(full_device_name); + return get_supported_property(get_plugin(parsed._deviceName), user_properties, keep_core_property, rw_only); +} + +ov::AnyMap ov::CoreImpl::get_hw_plugin_properties_or_forward(const Plugin& plugin, + const AnyMap& config, + bool rw_only) const { + constexpr auto keep_core = false; + return is_virtual_device(plugin.get_name()) ? config : get_supported_property(plugin, config, keep_core, rw_only); +} + ov::SoPtr ov::CoreImpl::get_default_context(const std::string& device_name) const { auto parsed = ov::parseDeviceNameIntoConfig(device_name); return get_plugin(parsed._deviceName).get_default_context(parsed._config); @@ -1684,17 +1691,3 @@ std::map ov::CoreImpl::get_versions(const std::string& return versions; } - -ov::AnyMap ov::CoreImpl::get_device_properties(const std::string& device_name, const AnyMap& config) const { - const auto parsed = parseDeviceNameIntoConfig(device_name, config); - return is_virtual_device(parsed._deviceName) - ? parsed._config - : split_config(parsed._config, get_supported_property(parsed._deviceName, parsed._config, false)).first; -} - -ov::AnyMap ov::CoreImpl::get_device_unsupported_properties(const std::string& device_name, const AnyMap& config) const { - const auto parsed = parseDeviceNameIntoConfig(device_name, config); - return is_virtual_device(parsed._deviceName) - ? AnyMap{} - : split_config(parsed._config, get_supported_property(parsed._deviceName, parsed._config, false)).second; -} diff --git a/src/inference/src/dev/core_impl.hpp b/src/inference/src/dev/core_impl.hpp index 297f703c5cf926..3233a575ae1bf8 100644 --- a/src/inference/src/dev/core_impl.hpp +++ b/src/inference/src/dev/core_impl.hpp @@ -183,6 +183,12 @@ class CoreImpl : public ov::ICore, public std::enable_shared_from_this& extensions) const; + Plugin get_plugin(const std::string& pluginName, bool on_create_filter_config) const; + + AnyMap get_supported_property(const Plugin& plugin, const AnyMap& config, bool keep_core, bool rw_only) const; + + AnyMap get_hw_plugin_properties_or_forward(const Plugin& plugin, const AnyMap& config, bool rw_only = false) const; + public: CoreImpl(); @@ -288,7 +294,9 @@ class CoreImpl : public ov::ICore, public std::enable_shared_from_this create_context(const std::string& device_name, const AnyMap& args) const override; - ov::AnyMap get_supported_property(const std::string& device_name, const ov::AnyMap& config, const bool keep_core_property = true) const override; + ov::AnyMap get_supported_property(const std::string& device_name, + const ov::AnyMap& config, + const bool keep_core_property = true) const override; ov::SoPtr get_default_context(const std::string& device_name) const override; @@ -317,24 +325,6 @@ class CoreImpl : public ov::ICore, public std::enable_shared_from_thiscompile_model(function, target_device, config)); - } else { - EXPECT_ANY_THROW(auto execNet = core->compile_model(function, target_device, config)); - } + + EXPECT_NO_THROW(auto execNet = core->compile_model(function, target_device, config)); } typedef std::tuple Date: Sun, 13 Oct 2024 19:38:20 +0000 Subject: [PATCH 04/21] Remove not used variable --- .../include/behavior/compiled_model/compiled_model_base.hpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/tests/functional/plugin/shared/include/behavior/compiled_model/compiled_model_base.hpp b/src/tests/functional/plugin/shared/include/behavior/compiled_model/compiled_model_base.hpp index af75c5013fad0f..35e0a4ac4ec25a 100644 --- a/src/tests/functional/plugin/shared/include/behavior/compiled_model/compiled_model_base.hpp +++ b/src/tests/functional/plugin/shared/include/behavior/compiled_model/compiled_model_base.hpp @@ -628,9 +628,6 @@ TEST_P(OVCompiledModelBaseTest, canLoadCorrectNetworkToGetExecutableWithIncorrec for (const auto& confItem : configuration) { config.emplace(confItem.first, confItem.second); } - bool is_meta_devices = target_device.find("AUTO") != std::string::npos || - target_device.find("MULTI") != std::string::npos || - target_device.find("HETERO") != std::string::npos; EXPECT_NO_THROW(auto execNet = core->compile_model(function, target_device, config)); } From b1387b22e665e702efb3717e47e1849d6a900aaf Mon Sep 17 00:00:00 2001 From: Pawel Raasz Date: Sun, 13 Oct 2024 19:59:00 +0000 Subject: [PATCH 05/21] Fix MSVC build error --- src/inference/src/dev/core_impl.cpp | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/inference/src/dev/core_impl.cpp b/src/inference/src/dev/core_impl.cpp index 741cf4bdbf0193..721ea5afbb3485 100644 --- a/src/inference/src/dev/core_impl.cpp +++ b/src/inference/src/dev/core_impl.cpp @@ -994,13 +994,10 @@ ov::AnyMap ov::CoreImpl::get_supported_property(const Plugin& plugin, supported_config_keys = core_level_properties; } - const auto may_add_supported = rw_only ? [](const PropertyName& p){return p.is_mutable();} : [](const - PropertyName&){return true;}; - // try to search against OV API 2.0' mutable supported_properties try { for (auto&& property : plugin.get_property(ov::supported_properties)) { - if (may_add_supported(property)) { + if (!rw_only || property.is_mutable()) { supported_config_keys.emplace_back(std::move(property)); } } @@ -1010,7 +1007,7 @@ ov::AnyMap ov::CoreImpl::get_supported_property(const Plugin& plugin, // try to search against internal supported_properties try { for (auto&& property : plugin.get_property(ov::internal::supported_properties)) { - if (may_add_supported(property)) { + if (!rw_only || property.is_mutable()) { supported_config_keys.emplace_back(std::move(property)); } } From 4f33336b89a83126fb50d2baa797195de89e7136 Mon Sep 17 00:00:00 2001 From: Pawel Raasz Date: Mon, 14 Oct 2024 06:35:00 +0000 Subject: [PATCH 06/21] Auto-Batch remove prop validation to meta device --- src/inference/src/dev/core_impl.cpp | 46 ++++++++++--------- src/plugins/auto_batch/src/plugin.cpp | 8 ---- src/plugins/intel_cpu/src/plugin.cpp | 3 +- .../compiled_model/compiled_model_base.hpp | 2 +- 4 files changed, 28 insertions(+), 31 deletions(-) diff --git a/src/inference/src/dev/core_impl.cpp b/src/inference/src/dev/core_impl.cpp index 721ea5afbb3485..f758ac0a2c6c40 100644 --- a/src/inference/src/dev/core_impl.cpp +++ b/src/inference/src/dev/core_impl.cpp @@ -957,24 +957,6 @@ ov::AnyMap ov::CoreImpl::get_supported_property(const Plugin& plugin, bool keep_core, bool rw_only) const { const auto& full_device_name = plugin.get_name(); - if (is_virtual_device(full_device_name)) { - // Considerations: - // 1. in case of virtual devices all the magic will happen on the level when - // virtual device calls ICore::get_supported_property for real HW devices - // so, for now we can return user properties almost as is without any - // filtering / flattening - // 2. The only exception here: while common properties like ov::num::streams or - // ov::hint::performance_mode are shared across all the devices, the - // ov::device::priority cannot be shared, because it's specific for current virtual - // plugin. So, we need to remove ov::device::priorities from the list, because it's - // supposed to be set for current virtual plugin and cannot be propagated down - auto return_properties = config; - auto device_priorities_it = return_properties.find(ov::device::priorities.name()); - if (device_priorities_it != return_properties.end()) { - return_properties.erase(device_priorities_it); - } - return return_properties; - } static const std::vector core_level_properties = { ov::cache_dir.name(), @@ -1028,15 +1010,37 @@ ov::AnyMap ov::CoreImpl::get_supported_property(const Plugin& plugin, ov::AnyMap ov::CoreImpl::get_supported_property(const std::string& full_device_name, const ov::AnyMap& user_properties, const bool keep_core_property) const { - const auto parsed = ov::parseDeviceNameIntoConfig(full_device_name); - return get_supported_property(get_plugin(parsed._deviceName), user_properties, keep_core_property, rw_only); + if (is_virtual_device(full_device_name)) { + // Considerations: + // 1. in case of virtual devices all the magic will happen on the level when + // virtual device calls ICore::get_supported_property for real HW devices + // so, for now we can return user properties almost as is without any + // filtering / flattening + // 2. The only exception here: while common properties like ov::num::streams or + // ov::hint::performance_mode are shared across all the devices, the + // ov::device::priority cannot be shared, because it's specific for current virtual + // plugin. So, we need to remove ov::device::priorities from the list, because it's + // supposed to be set for current virtual plugin and cannot be propagated down + auto return_properties = user_properties; + auto device_priorities_it = return_properties.find(ov::device::priorities.name()); + if (device_priorities_it != return_properties.end()) { + return_properties.erase(device_priorities_it); + } + return return_properties; + } else { + const auto parsed = ov::parseDeviceNameIntoConfig(full_device_name); + return get_supported_property(get_plugin(parsed._deviceName), user_properties, keep_core_property, rw_only); + } } ov::AnyMap ov::CoreImpl::get_hw_plugin_properties_or_forward(const Plugin& plugin, const AnyMap& config, bool rw_only) const { constexpr auto keep_core = false; - return is_virtual_device(plugin.get_name()) ? config : get_supported_property(plugin, config, keep_core, rw_only); + const auto& device_name = plugin.get_name(); + const auto forward_config = is_virtual_device(device_name) || is_proxy_device(device_name); + + return forward_config ? config : get_supported_property(plugin, config, keep_core, rw_only); } ov::SoPtr ov::CoreImpl::get_default_context(const std::string& device_name) const { diff --git a/src/plugins/auto_batch/src/plugin.cpp b/src/plugins/auto_batch/src/plugin.cpp index 2e3e17cd43086d..832adf9cc964f7 100644 --- a/src/plugins/auto_batch/src/plugin.cpp +++ b/src/plugins/auto_batch/src/plugin.cpp @@ -51,14 +51,6 @@ DeviceInformation Plugin::parse_meta_device(const std::string& devices_batch_con const ov::AnyMap& user_config) const { auto meta_device = parse_batch_device(devices_batch_config); meta_device.device_config = get_core()->get_supported_property(meta_device.device_name, user_config); - // check that no irrelevant config-keys left - for (const auto& k : user_config) { - const auto& name = k.first; - if (meta_device.device_config.find(name) == meta_device.device_config.end() && - !ov::util::contains(supported_configKeys, name)) { - OPENVINO_THROW("Unsupported config key: ", name); - } - } return meta_device; } diff --git a/src/plugins/intel_cpu/src/plugin.cpp b/src/plugins/intel_cpu/src/plugin.cpp index f816ced6071e33..9aceb6e0537787 100644 --- a/src/plugins/intel_cpu/src/plugin.cpp +++ b/src/plugins/intel_cpu/src/plugin.cpp @@ -196,7 +196,7 @@ static Config::ModelType getModelType(const std::shared_ptr& model) if (op::util::has_op_with_type(model) || op::util::has_op_with_type(model)) return Config::ModelType::CNN; - + if (op::util::has_op_with_type(model) && model->get_variables().size() > 0) return Config::ModelType::LLM; @@ -446,6 +446,7 @@ ov::Any Plugin::get_ro_property(const std::string& name, const ov::AnyMap& optio ov::PropertyName{ov::internal::caching_properties.name(), ov::PropertyMutability::RO}, ov::PropertyName{ov::internal::exclusive_async_requests.name(), ov::PropertyMutability::RW}, ov::PropertyName{ov::internal::compiled_model_runtime_properties.name(), ov::PropertyMutability::RO}, + ov::PropertyName{ov::intel_cpu::snippets_mode.name(), ov::PropertyMutability::WO}, ov::PropertyName{ov::internal::compiled_model_runtime_properties_supported.name(), ov::PropertyMutability::RO}, ov::PropertyName{ov::cache_encryption_callbacks.name(), ov::PropertyMutability::WO}}; diff --git a/src/tests/functional/plugin/shared/include/behavior/compiled_model/compiled_model_base.hpp b/src/tests/functional/plugin/shared/include/behavior/compiled_model/compiled_model_base.hpp index 35e0a4ac4ec25a..3445b16878f050 100644 --- a/src/tests/functional/plugin/shared/include/behavior/compiled_model/compiled_model_base.hpp +++ b/src/tests/functional/plugin/shared/include/behavior/compiled_model/compiled_model_base.hpp @@ -629,7 +629,7 @@ TEST_P(OVCompiledModelBaseTest, canLoadCorrectNetworkToGetExecutableWithIncorrec config.emplace(confItem.first, confItem.second); } - EXPECT_NO_THROW(auto execNet = core->compile_model(function, target_device, config)); + OV_ASSERT_NO_THROW(std::ignore = core->compile_model(function, target_device, config)); } typedef std::tuple Date: Mon, 14 Oct 2024 07:25:55 +0000 Subject: [PATCH 07/21] Add missing supported properties in Template --- src/plugins/template/src/plugin.cpp | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/src/plugins/template/src/plugin.cpp b/src/plugins/template/src/plugin.cpp index ee885f67e188b5..04515eee94050e 100644 --- a/src/plugins/template/src/plugin.cpp +++ b/src/plugins/template/src/plugin.cpp @@ -256,15 +256,18 @@ ov::Any ov::template_plugin::Plugin::get_property(const std::string& name, const return ro_properties; }; const auto& default_rw_properties = []() { - std::vector rw_properties{ov::device::id, - ov::enable_profiling, - ov::hint::performance_mode, - ov::hint::num_requests, - ov::hint::inference_precision, - ov::hint::execution_mode, - ov::num_streams, - ov::template_plugin::disable_transformations, - ov::log::level}; + std::vector rw_properties{ + ov::device::id, + ov::enable_profiling, + ov::hint::performance_mode, + ov::hint::num_requests, + ov::hint::inference_precision, + ov::hint::execution_mode, + ov::num_streams, + ov::template_plugin::disable_transformations, + ov::log::level, + ov::hint::model_priority, + }; return rw_properties; }; if (ov::supported_properties == name) { From ce4a5a09c4c00a4e5fbdc92daacaa7ba2c54c8bb Mon Sep 17 00:00:00 2001 From: Pawel Raasz Date: Mon, 14 Oct 2024 07:26:52 +0000 Subject: [PATCH 08/21] Add parse only properties in auto-batch --- src/plugins/auto_batch/src/plugin.cpp | 20 +++++++++++++++++--- src/plugins/auto_batch/src/plugin.hpp | 4 +++- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/src/plugins/auto_batch/src/plugin.cpp b/src/plugins/auto_batch/src/plugin.cpp index 832adf9cc964f7..3e1b35fe338612 100644 --- a/src/plugins/auto_batch/src/plugin.cpp +++ b/src/plugins/auto_batch/src/plugin.cpp @@ -47,13 +47,27 @@ DeviceInformation Plugin::parse_batch_device(const std::string& device_with_batc return {std::move(deviceName), {{}}, static_cast(batch)}; } -DeviceInformation Plugin::parse_meta_device(const std::string& devices_batch_config, - const ov::AnyMap& user_config) const { +DeviceInformation Plugin::parse_only_meta_device(const std::string& devices_batch_config, + const ov::AnyMap& user_config) const { auto meta_device = parse_batch_device(devices_batch_config); meta_device.device_config = get_core()->get_supported_property(meta_device.device_name, user_config); return meta_device; } +DeviceInformation Plugin::parse_meta_device(const std::string& devices_batch_config, + const ov::AnyMap& user_config) const { + auto meta_device = parse_only_meta_device(devices_batch_config, user_config); + // check that no irrelevant config-keys left + for (const auto& k : user_config) { + const auto& name = k.first; + if (meta_device.device_config.find(name) == meta_device.device_config.end() && + !ov::util::contains(supported_configKeys, name)) { + OPENVINO_THROW("Unsupported config key: ", name); + } + } + return meta_device; +} + ov::SoPtr Plugin::create_context(const ov::AnyMap& remote_properties) const { auto full_properties = remote_properties; auto it = full_properties.find(ov::device::priorities.name()); @@ -132,7 +146,7 @@ std::shared_ptr Plugin::compile_model(const std::shared_ptr< if (device_batch == full_properties.end()) { OPENVINO_THROW("ov::device::priorities key for AUTO BATCH is not set for BATCH device"); } - auto meta_device = parse_meta_device(device_batch->second.as(), properties); + auto meta_device = parse_only_meta_device(device_batch->second.as(), properties); const auto& device_name = meta_device.device_name; const auto& device_config = meta_device.device_config; diff --git a/src/plugins/auto_batch/src/plugin.hpp b/src/plugins/auto_batch/src/plugin.hpp index 37a777cc970b6a..ff278be6adea57 100644 --- a/src/plugins/auto_batch/src/plugin.hpp +++ b/src/plugins/auto_batch/src/plugin.hpp @@ -60,6 +60,8 @@ class Plugin : public ov::IPlugin { protected: #endif + DeviceInformation parse_only_meta_device(const std::string& devices_batch_config, + const ov::AnyMap& user_config) const; DeviceInformation parse_meta_device(const std::string& devices_batch_config, const ov::AnyMap& user_config) const; static DeviceInformation parse_batch_device(const std::string& device_with_batch); @@ -68,4 +70,4 @@ class Plugin : public ov::IPlugin { mutable ov::AnyMap m_plugin_config; }; } // namespace autobatch_plugin -} // namespace ov \ No newline at end of file +} // namespace ov From 945aab22742a57ecc580a7f0bc210cc1b0601144 Mon Sep 17 00:00:00 2001 From: Pawel Raasz Date: Mon, 14 Oct 2024 09:57:06 +0000 Subject: [PATCH 09/21] Template plugin sync supported properties --- src/plugins/template/src/plugin.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/plugins/template/src/plugin.cpp b/src/plugins/template/src/plugin.cpp index 04515eee94050e..cf1f764f820a74 100644 --- a/src/plugins/template/src/plugin.cpp +++ b/src/plugins/template/src/plugin.cpp @@ -282,7 +282,10 @@ ov::Any ov::template_plugin::Plugin::get_property(const std::string& name, const } else if (ov::internal::supported_properties == name) { return decltype(ov::internal::supported_properties)::value_type{ ov::PropertyName{ov::internal::caching_properties.name(), ov::PropertyMutability::RO}, - ov::PropertyName{ov::internal::exclusive_async_requests.name(), ov::PropertyMutability::RW}}; + ov::PropertyName{ov::internal::exclusive_async_requests.name(), ov::PropertyMutability::RW}, + ov::PropertyName{ov::inference_num_threads.name(), ov::PropertyMutability::RW}, + ov::PropertyName{ov::internal::threads_per_stream.name(), ov::PropertyMutability::RW}, + ov::PropertyName{ov::cache_encryption_callbacks.name(), ov::PropertyMutability::WO}}; } else if (ov::available_devices == name) { // TODO: fill list of available devices return decltype(ov::available_devices)::value_type{{""}}; From a89eed8cff6e792a0edc585d7155abd18570e31c Mon Sep 17 00:00:00 2001 From: Pawel Raasz Date: Mon, 14 Oct 2024 10:09:39 +0000 Subject: [PATCH 10/21] CPU plugin sync supported properties --- src/plugins/intel_cpu/src/plugin.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/plugins/intel_cpu/src/plugin.cpp b/src/plugins/intel_cpu/src/plugin.cpp index 588412f6ea4405..6c2a9c66deaa5d 100644 --- a/src/plugins/intel_cpu/src/plugin.cpp +++ b/src/plugins/intel_cpu/src/plugin.cpp @@ -451,10 +451,14 @@ ov::Any Plugin::get_ro_property(const std::string& name, const ov::AnyMap& optio #endif ov::PropertyName{ov::internal::exclusive_async_requests.name(), ov::PropertyMutability::RW}, ov::PropertyName{ov::internal::compiled_model_runtime_properties.name(), ov::PropertyMutability::RO}, + // as write-only as internal only ov::PropertyName{ov::intel_cpu::snippets_mode.name(), ov::PropertyMutability::WO}, ov::PropertyName{ov::internal::compiled_model_runtime_properties_supported.name(), - ov::PropertyMutability::RO}, - ov::PropertyName{ov::cache_encryption_callbacks.name(), ov::PropertyMutability::WO}}; + ov::PropertyMutability::RO}, + ov::PropertyName{ov::cache_encryption_callbacks.name(), ov::PropertyMutability::WO}, + ov::PropertyName{ov::intel_cpu::lp_transforms_mode.name(), ov::PropertyMutability::WO}, + ov::PropertyName{ov::intel_cpu::cpu_runtime_cache_capacity.name(), ov::PropertyMutability::WO}, + }; } else if (name == ov::device::full_name) { return decltype(ov::device::full_name)::value_type(deviceFullName); } else if (name == ov::available_devices) { From 0426bdade0c1b7649e060fcb7cd00aa318a70ac0 Mon Sep 17 00:00:00 2001 From: Pawel Raasz Date: Mon, 14 Oct 2024 10:09:59 +0000 Subject: [PATCH 11/21] Hetero plugin sync supported properties --- src/plugins/hetero/src/plugin.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/plugins/hetero/src/plugin.cpp b/src/plugins/hetero/src/plugin.cpp index 298be6fa201f4f..0f2410e28660f9 100644 --- a/src/plugins/hetero/src/plugin.cpp +++ b/src/plugins/hetero/src/plugin.cpp @@ -283,7 +283,10 @@ ov::Any ov::hetero::Plugin::get_property(const std::string& name, const ov::AnyM return decltype(ov::supported_properties)::value_type(std::move(supported_properties)); } else if (ov::internal::supported_properties == name) { return decltype(ov::internal::supported_properties)::value_type{ - ov::PropertyName{ov::internal::caching_properties.name(), ov::PropertyMutability::RO}}; + ov::PropertyName{ov::internal::caching_properties.name(), ov::PropertyMutability::RO}, + // write-only as internal + ov::PropertyName{ov::cache_encryption_callbacks.name(), ov::PropertyMutability::WO}, + }; } else if (ov::device::full_name == name) { return decltype(ov::device::full_name)::value_type{get_device_name()}; } else if (ov::internal::caching_properties == name) { From 8ae4d36593dbb3caa66c4e1041672f15c135478e Mon Sep 17 00:00:00 2001 From: Pawel Raasz Date: Mon, 14 Oct 2024 10:27:27 +0000 Subject: [PATCH 12/21] Make `parse_only_meta_device` member private --- src/plugins/auto_batch/src/plugin.hpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/plugins/auto_batch/src/plugin.hpp b/src/plugins/auto_batch/src/plugin.hpp index ff278be6adea57..8bfdb7c4e8385d 100644 --- a/src/plugins/auto_batch/src/plugin.hpp +++ b/src/plugins/auto_batch/src/plugin.hpp @@ -60,13 +60,14 @@ class Plugin : public ov::IPlugin { protected: #endif - DeviceInformation parse_only_meta_device(const std::string& devices_batch_config, - const ov::AnyMap& user_config) const; DeviceInformation parse_meta_device(const std::string& devices_batch_config, const ov::AnyMap& user_config) const; static DeviceInformation parse_batch_device(const std::string& device_with_batch); private: + DeviceInformation parse_only_meta_device(const std::string& devices_batch_config, + const ov::AnyMap& user_config) const; + mutable ov::AnyMap m_plugin_config; }; } // namespace autobatch_plugin From 327c93acee8985448f714bf4fcb31a1f9cf7d56b Mon Sep 17 00:00:00 2001 From: Pawel Raasz Date: Tue, 22 Oct 2024 09:35:47 +0000 Subject: [PATCH 13/21] Remove not used parameter in function --- src/inference/src/dev/core_impl.cpp | 6 ++---- src/inference/src/dev/core_impl.hpp | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/inference/src/dev/core_impl.cpp b/src/inference/src/dev/core_impl.cpp index f3c7a8b9d1ff3b..7f0d7a6eda1c2a 100644 --- a/src/inference/src/dev/core_impl.cpp +++ b/src/inference/src/dev/core_impl.cpp @@ -1034,14 +1034,12 @@ ov::AnyMap ov::CoreImpl::get_supported_property(const std::string& full_device_n } } -ov::AnyMap ov::CoreImpl::get_hw_plugin_properties_or_forward(const Plugin& plugin, - const AnyMap& config, - bool rw_only) const { +ov::AnyMap ov::CoreImpl::get_hw_plugin_properties_or_forward(const Plugin& plugin, const AnyMap& config) const { constexpr auto keep_core = false; const auto& device_name = plugin.get_name(); const auto forward_config = is_virtual_device(device_name) || is_proxy_device(device_name); - return forward_config ? config : get_supported_property(plugin, config, keep_core, rw_only); + return forward_config ? config : get_supported_property(plugin, config, keep_core, !rw_only); } ov::SoPtr ov::CoreImpl::get_default_context(const std::string& device_name) const { diff --git a/src/inference/src/dev/core_impl.hpp b/src/inference/src/dev/core_impl.hpp index 281f05ee7c886f..79990bc48add9a 100644 --- a/src/inference/src/dev/core_impl.hpp +++ b/src/inference/src/dev/core_impl.hpp @@ -187,7 +187,7 @@ class CoreImpl : public ov::ICore, public std::enable_shared_from_this Date: Wed, 20 Nov 2024 13:53:59 +0000 Subject: [PATCH 14/21] Add kv_cache_precision as supported property by GPU to forward it after config filter by device Signed-off-by: Pawel Raasz --- src/plugins/intel_gpu/src/plugin/plugin.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/plugins/intel_gpu/src/plugin/plugin.cpp b/src/plugins/intel_gpu/src/plugin/plugin.cpp index c3ba90fd66f7a8..4bf93f51f2d48f 100644 --- a/src/plugins/intel_gpu/src/plugin/plugin.cpp +++ b/src/plugins/intel_gpu/src/plugin/plugin.cpp @@ -598,6 +598,7 @@ std::vector Plugin::get_supported_properties() const { ov::PropertyName{ov::hint::dynamic_quantization_group_size.name(), PropertyMutability::RW}, ov::PropertyName{ov::hint::activations_scale_factor.name(), PropertyMutability::RW}, ov::PropertyName{ov::weights_path.name(), PropertyMutability::RW}, + ov::PropertyName{ov::hint::kv_cache_precision.name(), PropertyMutability::RW}, }; return supported_properties; From 2772f3d4e992189211ae8b71c872e75a958f6c96 Mon Sep 17 00:00:00 2001 From: "Raasz, Pawel" Date: Tue, 10 Dec 2024 12:42:36 +0000 Subject: [PATCH 15/21] Filter core properties by using local core config Signed-off-by: Raasz, Pawel --- src/inference/src/dev/core_impl.cpp | 251 ++++++++---------- src/inference/src/dev/core_impl.hpp | 19 +- src/plugins/hetero/src/plugin.cpp | 2 - src/plugins/intel_cpu/src/plugin.cpp | 7 - src/plugins/template/src/plugin.cpp | 3 +- .../compiled_model/compiled_model_base.hpp | 9 +- 6 files changed, 129 insertions(+), 162 deletions(-) diff --git a/src/inference/src/dev/core_impl.cpp b/src/inference/src/dev/core_impl.cpp index 4590c710646880..25dde8973d9356 100644 --- a/src/inference/src/dev/core_impl.cpp +++ b/src/inference/src/dev/core_impl.cpp @@ -38,6 +38,18 @@ ov::ICore::~ICore() = default; +namespace ov { +namespace util { +template +constexpr std::array< + typename std::conditional::value, typename std::common_type::type, T>::type, + sizeof...(Args)> +make_array(Args&&... args) { + return {std::forward(args)...}; +} +} // namespace util +} // namespace ov + namespace { #ifdef PROXY_PLUGIN_ENABLED @@ -206,7 +218,13 @@ void clean_batch_properties(const std::string& deviceName, ov::AnyMap& config, c } } -constexpr bool rw_only = true; +constexpr bool keep_core = true; + +static const auto core_properties_names = + ov::util::make_array(ov::cache_dir.name(), ov::enable_mmap.name(), ov::force_tbb_terminate.name()); + +static const auto auto_batch_properties_names = + ov::util::make_array(ov::auto_batch_timeout.name(), ov::hint::allow_auto_batching.name()); } // namespace bool ov::is_config_applicable(const std::string& user_device_name, const std::string& subprop_device_name) { @@ -306,8 +324,9 @@ ov::Parsed ov::parseDeviceNameIntoConfig(const std::string& deviceName, // keep batch property only when called from query_supported_property if (!keep_core_property) { - clean_batch_properties(updated_device_name, updated_config, ov::hint::allow_auto_batching); - clean_batch_properties(updated_device_name, updated_config, ov::auto_batch_timeout); + for (const auto& name : auto_batch_properties_names) { + clean_batch_properties(updated_device_name, updated_config, name); + } } return {std::move(updated_device_name), std::move(updated_config)}; @@ -546,8 +565,8 @@ void ov::CoreImpl::register_plugins_in_registry(const std::string& xml_config_fi } } -ov::Plugin ov::CoreImpl::get_plugin(const std::string& pluginName, bool on_create_filter_config) const { - OV_ITT_SCOPE(FIRST_INFERENCE, ov::itt::domains::LoadTime, "CoreImpl::get_plugin::supported_config"); +ov::Plugin ov::CoreImpl::get_plugin(const std::string& pluginName) const { + OV_ITT_SCOPE(FIRST_INFERENCE, ov::itt::domains::LoadTime, "CoreImpl::get_plugin"); auto deviceName = pluginName; if (deviceName == ov::DEFAULT_DEVICE_NAME) @@ -665,8 +684,7 @@ ov::Plugin ov::CoreImpl::get_plugin(const std::string& pluginName, bool on_creat { OPENVINO_SUPPRESS_DEPRECATED_START if (device_supports_cache_dir(plugin)) { - ov::AnyMap empty_map; - auto cacheConfig = coreConfig.get_cache_config_for_device(plugin, empty_map); + auto cacheConfig = coreConfig.get_cache_config_for_device(plugin); if (cacheConfig._cacheManager) { desc.defaultConfig[ov::cache_dir.name()] = cacheConfig._cacheDir; } @@ -692,17 +710,11 @@ ov::Plugin ov::CoreImpl::get_plugin(const std::string& pluginName, bool on_creat ov::DeviceIDParser parser(pluginDesc.first); if (pluginDesc.first.find(deviceName) != std::string::npos && !parser.get_device_id().empty()) { pluginDesc.second.defaultConfig[deviceKey] = parser.get_device_id(); - plugin.set_property( - on_create_filter_config - ? get_hw_plugin_properties_or_forward(plugin, pluginDesc.second.defaultConfig) - : pluginDesc.second.defaultConfig); + plugin.set_property(pluginDesc.second.defaultConfig); } } } - - plugin.set_property(on_create_filter_config - ? get_hw_plugin_properties_or_forward(plugin, desc.defaultConfig) - : desc.defaultConfig); + plugin.set_property(desc.defaultConfig); }); } @@ -734,12 +746,6 @@ ov::Plugin ov::CoreImpl::get_plugin(const std::string& pluginName, bool on_creat } } -ov::Plugin ov::CoreImpl::get_plugin(const std::string& pluginName) const { - OV_ITT_SCOPE(FIRST_INFERENCE, ov::itt::domains::LoadTime, "CoreImpl::get_plugin"); - constexpr auto on_plugin_create_filter_config = false; - return get_plugin(pluginName, on_plugin_create_filter_config); -} - ov::SoPtr ov::CoreImpl::compile_model(const std::shared_ptr& model_, const std::string& device_name, const ov::AnyMap& config) const { @@ -749,24 +755,27 @@ ov::SoPtr ov::CoreImpl::compile_model(const std::shared_ptr< // if auto-batching is applicable, the below function will patch the device name and config accordingly: auto model = apply_auto_batching(model_, deviceName, config_with_batch); - auto parsed = parseDeviceNameIntoConfig(deviceName, config_with_batch, is_proxy_device(device_name)); - auto plugin = get_plugin(parsed._deviceName, !is_virtual_device(parsed._deviceName)); + auto local_core_config = coreConfig; + auto parsed = parseDeviceNameIntoConfig(deviceName, config_with_batch, is_proxy_device(deviceName)); + local_core_config.set_and_update(parsed._config); + auto plugin = get_plugin(parsed._deviceName); + ov::SoPtr res; - auto cacheManager = coreConfig.get_cache_config_for_device(plugin, parsed._config)._cacheManager; + auto cacheManager = local_core_config.get_cache_config_for_device(plugin)._cacheManager; // Skip caching for proxy plugin. HW plugin will load network from the cache if (cacheManager && device_supports_model_caching(plugin) && !is_proxy_device(plugin)) { - CacheContent cacheContent{cacheManager}; + CacheContent cacheContent{cacheManager, local_core_config.get_enable_mmap()}; cacheContent.blobId = ov::ModelCache::compute_hash(model, create_compile_config(plugin, parsed._config)); std::unique_ptr lock = cacheGuard.get_hash_lock(cacheContent.blobId); res = load_model_from_cache(cacheContent, plugin, parsed._config, ov::SoPtr{}, [&]() { return compile_model_and_cache(plugin, model, - get_hw_plugin_properties_or_forward(plugin, parsed._config), + parsed._config, ov::SoPtr{}, cacheContent); }); } else { - res = plugin.compile_model(model, get_hw_plugin_properties_or_forward(plugin, parsed._config)); + res = plugin.compile_model(model, parsed._config); } return res; } @@ -782,24 +791,23 @@ ov::SoPtr ov::CoreImpl::compile_model(const std::shared_ptr< // if auto-batching is applicable, the below function will patch the device name and config accordingly: auto model = apply_auto_batching(model_, deviceName, config_with_batch); + auto local_core_config = coreConfig; auto parsed = parseDeviceNameIntoConfig(deviceName, config_with_batch, is_proxy_device(deviceName)); - auto plugin = get_plugin(parsed._deviceName, !is_virtual_device(parsed._deviceName)); + local_core_config.set_and_update(parsed._config); + auto plugin = get_plugin(parsed._deviceName); + ov::SoPtr res; - auto cacheManager = coreConfig.get_cache_config_for_device(plugin, parsed._config)._cacheManager; + auto cacheManager = local_core_config.get_cache_config_for_device(plugin)._cacheManager; // Skip caching for proxy plugin. HW plugin will load network from the cache if (cacheManager && device_supports_model_caching(plugin) && !is_proxy_device(plugin)) { - CacheContent cacheContent{cacheManager}; + CacheContent cacheContent{cacheManager, local_core_config.get_enable_mmap()}; cacheContent.blobId = ov::ModelCache::compute_hash(model, create_compile_config(plugin, parsed._config)); std::unique_ptr lock = cacheGuard.get_hash_lock(cacheContent.blobId); res = load_model_from_cache(cacheContent, plugin, parsed._config, context, [&]() { - return compile_model_and_cache(plugin, - model, - get_hw_plugin_properties_or_forward(plugin, parsed._config), - context, - cacheContent); + return compile_model_and_cache(plugin, model, parsed._config, context, cacheContent); }); } else { - res = plugin.compile_model(model, context, get_hw_plugin_properties_or_forward(plugin, parsed._config)); + res = plugin.compile_model(model, context, parsed._config); } return res; } @@ -808,28 +816,27 @@ ov::SoPtr ov::CoreImpl::compile_model(const std::string& mod const std::string& device_name, const ov::AnyMap& config) const { OV_ITT_SCOPE(FIRST_INFERENCE, ov::itt::domains::LoadTime, "Core::compile_model::Path"); - auto parsed = parseDeviceNameIntoConfig(device_name, config); + auto parsed = parseDeviceNameIntoConfig(device_name, config, keep_core); + auto local_core_config = coreConfig; // in case of compile_model(file_name), we need to clear-up core-level properties - auto plugin = get_plugin(parsed._deviceName, !is_virtual_device(parsed._deviceName)); - ov::SoPtr compiled_model; + local_core_config.set_and_update(parsed._config); + auto plugin = get_plugin(parsed._deviceName); - auto cacheManager = coreConfig.get_cache_config_for_device(plugin, parsed._config)._cacheManager; + ov::SoPtr compiled_model; + auto cacheManager = local_core_config.get_cache_config_for_device(plugin)._cacheManager; + // Skip caching for proxy plugin. HW plugin will load network from the cache if (cacheManager && device_supports_model_caching(plugin) && !is_proxy_device(plugin)) { - // Skip caching for proxy plugin. HW plugin will load network from the cache - CacheContent cacheContent{cacheManager, model_path}; + CacheContent cacheContent{cacheManager, local_core_config.get_enable_mmap(), model_path}; cacheContent.blobId = ov::ModelCache::compute_hash(model_path, create_compile_config(plugin, parsed._config)); std::unique_ptr lock = cacheGuard.get_hash_lock(cacheContent.blobId); compiled_model = load_model_from_cache(cacheContent, plugin, parsed._config, ov::SoPtr{}, [&]() { - auto model = read_model(model_path, std::string{}); - return compile_model_and_cache(plugin, - model, - get_hw_plugin_properties_or_forward(plugin, parsed._config), - {}, - cacheContent); + auto model = + ov::util::read_model(model_path, std::string{}, extensions, local_core_config.get_enable_mmap()); + return compile_model_and_cache(plugin, model, parsed._config, {}, cacheContent); }); } else { - compiled_model = plugin.compile_model(model_path, get_hw_plugin_properties_or_forward(plugin, parsed._config)); + compiled_model = plugin.compile_model(model_path, parsed._config); } return compiled_model; } @@ -839,15 +846,16 @@ ov::SoPtr ov::CoreImpl::compile_model(const std::string& mod const std::string& device_name, const ov::AnyMap& config) const { OV_ITT_SCOPED_TASK(ov::itt::domains::OV, "Core::compile_model::from_memory"); - auto parsed = parseDeviceNameIntoConfig(device_name, config); - // in case of compile_model(file_name), we need to clear-up core-level properties - auto plugin = get_plugin(parsed._deviceName, !is_virtual_device(parsed._deviceName)); - ov::SoPtr compiled_model; + auto parsed = parseDeviceNameIntoConfig(device_name, config, keep_core); + auto local_core_config = coreConfig; + local_core_config.set_and_update(parsed._config); + auto plugin = get_plugin(parsed._deviceName); - auto cacheManager = coreConfig.get_cache_config_for_device(plugin, parsed._config)._cacheManager; + ov::SoPtr compiled_model; + auto cacheManager = local_core_config.get_cache_config_for_device(plugin)._cacheManager; // Skip caching for proxy plugin. HW plugin will load network from the cache if (cacheManager && device_supports_model_caching(plugin) && !is_proxy_device(plugin)) { - CacheContent cacheContent{cacheManager}; + CacheContent cacheContent{cacheManager, local_core_config.get_enable_mmap()}; cacheContent.blobId = ov::ModelCache::compute_hash(model_str, weights, create_compile_config(plugin, parsed._config)); std::unique_ptr lock = cacheGuard.get_hash_lock(cacheContent.blobId); @@ -856,13 +864,13 @@ ov::SoPtr ov::CoreImpl::compile_model(const std::string& mod auto model = read_model(model_str, weights); return compile_model_and_cache(plugin, model, - get_hw_plugin_properties_or_forward(plugin, parsed._config), + parsed._config, ov::SoPtr{}, cacheContent); }); } else { auto model = read_model(model_str, weights); - compiled_model = plugin.compile_model(model, get_hw_plugin_properties_or_forward(plugin, parsed._config)); + compiled_model = plugin.compile_model(model, parsed._config); } return compiled_model; } @@ -953,34 +961,45 @@ ov::SoPtr ov::CoreImpl::create_context(const std::string& de return get_plugin(parsed._deviceName).create_context(parsed._config); } -ov::AnyMap ov::CoreImpl::get_supported_property(const Plugin& plugin, - const ov::AnyMap& config, - bool keep_core, - bool rw_only) const { - const auto& full_device_name = plugin.get_name(); - - static const std::vector core_level_properties = { - ov::cache_dir.name(), - ov::force_tbb_terminate.name(), - // auto-batch properties are also treated as core-level - ov::auto_batch_timeout.name(), - ov::hint::allow_auto_batching.name(), - }; +ov::AnyMap ov::CoreImpl::get_supported_property(const std::string& full_device_name, + const ov::AnyMap& user_properties, + const bool keep_core_property) const { + if (is_virtual_device(full_device_name)) { + // Considerations: + // 1. in case of virtual devices all the magic will happen on the level when + // virtual device calls ICore::get_supported_property for real HW devices + // so, for now we can return user properties almost as is without any + // filtering / flattening + // 2. The only exception here: while common properties like ov::num::streams or + // ov::hint::performance_mode are shared across all the devices, the + // ov::device::priority cannot be shared, because it's specific for current virtual + // plugin. So, we need to remove ov::device::priorities from the list, because it's + // supposed to be set for current virtual plugin and cannot be propagated down + auto return_properties = user_properties; + auto device_priorities_it = return_properties.find(ov::device::priorities.name()); + if (device_priorities_it != return_properties.end()) { + return_properties.erase(device_priorities_it); + } + return return_properties; + } - const auto flattened = ov::parseDeviceNameIntoConfig(full_device_name, config, true); + const auto flattened = parseDeviceNameIntoConfig(full_device_name, user_properties, keep_core_property); const auto& flattened_config = flattened._config; + const auto& device_name = flattened._deviceName; // virtual plugins should bypass core-level properties to HW plugins // so, we need to report them as supported std::vector supported_config_keys; if (keep_core) { - supported_config_keys = core_level_properties; + auto last = supported_config_keys.end(); + last = supported_config_keys.insert(last, core_properties_names.begin(), core_properties_names.end()); + supported_config_keys.insert(last, auto_batch_properties_names.begin(), auto_batch_properties_names.end()); } // try to search against OV API 2.0' mutable supported_properties try { - for (auto&& property : plugin.get_property(ov::supported_properties)) { - if (!rw_only || property.is_mutable()) { + for (auto&& property : ICore::get_property(device_name, ov::supported_properties)) { + if (property.is_mutable()) { supported_config_keys.emplace_back(std::move(property)); } } @@ -989,8 +1008,8 @@ ov::AnyMap ov::CoreImpl::get_supported_property(const Plugin& plugin, // try to search against internal supported_properties try { - for (auto&& property : plugin.get_property(ov::internal::supported_properties)) { - if (!rw_only || property.is_mutable()) { + for (auto&& property : ICore::get_property(device_name, ov::internal::supported_properties)) { + if (property.is_mutable()) { supported_config_keys.emplace_back(std::move(property)); } } @@ -1008,40 +1027,6 @@ ov::AnyMap ov::CoreImpl::get_supported_property(const Plugin& plugin, return supported_config; } -ov::AnyMap ov::CoreImpl::get_supported_property(const std::string& full_device_name, - const ov::AnyMap& user_properties, - const bool keep_core_property) const { - if (is_virtual_device(full_device_name)) { - // Considerations: - // 1. in case of virtual devices all the magic will happen on the level when - // virtual device calls ICore::get_supported_property for real HW devices - // so, for now we can return user properties almost as is without any - // filtering / flattening - // 2. The only exception here: while common properties like ov::num::streams or - // ov::hint::performance_mode are shared across all the devices, the - // ov::device::priority cannot be shared, because it's specific for current virtual - // plugin. So, we need to remove ov::device::priorities from the list, because it's - // supposed to be set for current virtual plugin and cannot be propagated down - auto return_properties = user_properties; - auto device_priorities_it = return_properties.find(ov::device::priorities.name()); - if (device_priorities_it != return_properties.end()) { - return_properties.erase(device_priorities_it); - } - return return_properties; - } else { - const auto parsed = ov::parseDeviceNameIntoConfig(full_device_name); - return get_supported_property(get_plugin(parsed._deviceName), user_properties, keep_core_property, rw_only); - } -} - -ov::AnyMap ov::CoreImpl::get_hw_plugin_properties_or_forward(const Plugin& plugin, const AnyMap& config) const { - constexpr auto keep_core = false; - const auto& device_name = plugin.get_name(); - const auto forward_config = is_virtual_device(device_name) || is_proxy_device(device_name); - - return forward_config ? config : get_supported_property(plugin, config, keep_core, !rw_only); -} - ov::SoPtr ov::CoreImpl::get_default_context(const std::string& device_name) const { auto parsed = ov::parseDeviceNameIntoConfig(device_name); return get_plugin(parsed._deviceName).get_default_context(parsed._config); @@ -1195,8 +1180,7 @@ ov::Any ov::CoreImpl::get_property(const std::string& device_name, if (parsed._deviceName.empty()) { return get_property_for_core(name); } else if (name == ov::cache_dir.name()) { - ov::AnyMap empty_map; - return coreConfig.get_cache_config_for_device(get_plugin(parsed._deviceName), empty_map)._cacheDir; + return coreConfig.get_cache_config_for_device(get_plugin(parsed._deviceName))._cacheDir; } return get_plugin(parsed._deviceName).get_property(name, parsed._config); } @@ -1334,9 +1318,7 @@ void ov::CoreImpl::set_property_for_device(const ov::AnyMap& configMap, const st { OPENVINO_SUPPRESS_DEPRECATED_START if (device_supports_cache_dir(plugin.second)) { - ov::AnyMap empty_map = {}; - configCopy[ov::cache_dir.name()] = - coreConfig.get_cache_config_for_device(plugin.second, empty_map)._cacheDir; + configCopy[ov::cache_dir.name()] = coreConfig.get_cache_config_for_device(plugin.second)._cacheDir; } else if (configCopy.count(ov::cache_dir.name()) > 0) { // Remove "CACHE_DIR" from config if it is not supported by plugin configCopy.erase(ov::cache_dir.name()); @@ -1446,8 +1428,8 @@ ov::SoPtr ov::CoreImpl::load_model_from_cache( try { cacheContent.cacheManager->read_cache_entry( cacheContent.blobId, - coreConfig.get_enable_mmap() && ov::util::contains(plugin.get_property(ov::internal::supported_properties), - ov::internal::caching_with_mmap), + cacheContent.mmap_enabled && ov::util::contains(plugin.get_property(ov::internal::supported_properties), + ov::internal::caching_with_mmap), [&](std::istream& networkStream) { OV_ITT_SCOPE(FIRST_INFERENCE, ov::itt::domains::LoadTime, @@ -1548,6 +1530,15 @@ ov::AnyMap ov::CoreImpl::create_compile_config(const ov::Plugin& plugin, const o return compile_config; } +ov::CoreImpl::CoreConfig::CoreConfig(const CoreConfig& other) { + { + std::lock_guard lock(other._cacheConfigMutex); + _cacheConfig = other._cacheConfig; + _cacheConfigPerDevice = other._cacheConfigPerDevice; + } + _flag_enable_mmap = other._flag_enable_mmap; +} + void ov::CoreImpl::CoreConfig::set_and_update(ov::AnyMap& config) { auto it = config.find(ov::cache_dir.name()); if (it != config.end()) { @@ -1590,30 +1581,10 @@ bool ov::CoreImpl::CoreConfig::get_enable_mmap() const { return _flag_enable_mmap; } -// Creating thread-safe copy of config including shared_ptr to ICacheManager -// Passing empty or not-existing name will return global cache config ov::CoreImpl::CoreConfig::CacheConfig ov::CoreImpl::CoreConfig::get_cache_config_for_device( - const ov::Plugin& plugin, - ov::AnyMap& parsedConfig) const { - // cache_dir is enabled locally in compile_model only - if (parsedConfig.count(ov::cache_dir.name())) { - const auto& cache_dir_val = parsedConfig.at(ov::cache_dir.name()).as(); - const auto& tempConfig = CoreConfig::CacheConfig::create(cache_dir_val); - // if plugin does not explicitly support cache_dir, and if plugin is not virtual, we need to remove - // it from config - if (!util::contains(plugin.get_property(ov::supported_properties), ov::cache_dir) && - !is_virtual_device(plugin.get_name())) { - parsedConfig.erase(ov::cache_dir.name()); - } - return tempConfig; - } else { // cache_dir is set to Core globally or for the specific device - std::lock_guard lock(_cacheConfigMutex); - if (_cacheConfigPerDevice.count(plugin.get_name()) > 0) { - return _cacheConfigPerDevice.at(plugin.get_name()); - } else { - return _cacheConfig; - } - } + const ov::Plugin& plugin) const { + std::lock_guard lock(_cacheConfigMutex); + return _cacheConfigPerDevice.count(plugin.get_name()) ? _cacheConfigPerDevice.at(plugin.get_name()) : _cacheConfig; } ov::CoreImpl::CoreConfig::CacheConfig ov::CoreImpl::CoreConfig::CacheConfig::create(const std::string& dir) { diff --git a/src/inference/src/dev/core_impl.hpp b/src/inference/src/dev/core_impl.hpp index 79990bc48add9a..ceaecb53c296de 100644 --- a/src/inference/src/dev/core_impl.hpp +++ b/src/inference/src/dev/core_impl.hpp @@ -63,6 +63,9 @@ class CoreImpl : public ov::ICore, public std::enable_shared_from_this _cacheManager; @@ -83,9 +86,8 @@ class CoreImpl : public ov::ICore, public std::enable_shared_from_this& cache_manager, + bool mmap_enabled = false, const std::string model_path = {}) : cacheManager(cache_manager), - modelPath(model_path) {} + modelPath(model_path), + mmap_enabled{mmap_enabled} {} std::shared_ptr cacheManager; std::string blobId = {}; std::string modelPath = {}; + bool mmap_enabled = false; }; // Core settings (cache config, etc) @@ -183,12 +188,6 @@ class CoreImpl : public ov::ICore, public std::enable_shared_from_this& extensions) const; - Plugin get_plugin(const std::string& pluginName, bool on_create_filter_config) const; - - AnyMap get_supported_property(const Plugin& plugin, const AnyMap& config, bool keep_core, bool rw_only) const; - - AnyMap get_hw_plugin_properties_or_forward(const Plugin& plugin, const AnyMap& config) const; - public: CoreImpl(); diff --git a/src/plugins/hetero/src/plugin.cpp b/src/plugins/hetero/src/plugin.cpp index 0f2410e28660f9..6d2740cfd05001 100644 --- a/src/plugins/hetero/src/plugin.cpp +++ b/src/plugins/hetero/src/plugin.cpp @@ -284,8 +284,6 @@ ov::Any ov::hetero::Plugin::get_property(const std::string& name, const ov::AnyM } else if (ov::internal::supported_properties == name) { return decltype(ov::internal::supported_properties)::value_type{ ov::PropertyName{ov::internal::caching_properties.name(), ov::PropertyMutability::RO}, - // write-only as internal - ov::PropertyName{ov::cache_encryption_callbacks.name(), ov::PropertyMutability::WO}, }; } else if (ov::device::full_name == name) { return decltype(ov::device::full_name)::value_type{get_device_name()}; diff --git a/src/plugins/intel_cpu/src/plugin.cpp b/src/plugins/intel_cpu/src/plugin.cpp index 7b33383df5c1f7..4e0f9f036e3c8e 100644 --- a/src/plugins/intel_cpu/src/plugin.cpp +++ b/src/plugins/intel_cpu/src/plugin.cpp @@ -453,13 +453,6 @@ ov::Any Plugin::get_ro_property(const std::string& name, const ov::AnyMap& optio #endif ov::PropertyName{ov::internal::exclusive_async_requests.name(), ov::PropertyMutability::RW}, ov::PropertyName{ov::internal::compiled_model_runtime_properties.name(), ov::PropertyMutability::RO}, - // as write-only as internal only - ov::PropertyName{ov::intel_cpu::snippets_mode.name(), ov::PropertyMutability::WO}, - ov::PropertyName{ov::internal::compiled_model_runtime_properties_supported.name(), - ov::PropertyMutability::RO}, - ov::PropertyName{ov::cache_encryption_callbacks.name(), ov::PropertyMutability::WO}, - ov::PropertyName{ov::intel_cpu::lp_transforms_mode.name(), ov::PropertyMutability::WO}, - ov::PropertyName{ov::intel_cpu::cpu_runtime_cache_capacity.name(), ov::PropertyMutability::WO}, }; } else if (name == ov::device::full_name) { return decltype(ov::device::full_name)::value_type(deviceFullName); diff --git a/src/plugins/template/src/plugin.cpp b/src/plugins/template/src/plugin.cpp index 40c2275ffcc6ac..20d72f0fad5a60 100644 --- a/src/plugins/template/src/plugin.cpp +++ b/src/plugins/template/src/plugin.cpp @@ -285,8 +285,7 @@ ov::Any ov::template_plugin::Plugin::get_property(const std::string& name, const ov::PropertyName{ov::internal::caching_properties.name(), ov::PropertyMutability::RO}, ov::PropertyName{ov::internal::exclusive_async_requests.name(), ov::PropertyMutability::RW}, ov::PropertyName{ov::inference_num_threads.name(), ov::PropertyMutability::RW}, - ov::PropertyName{ov::internal::threads_per_stream.name(), ov::PropertyMutability::RW}, - ov::PropertyName{ov::cache_encryption_callbacks.name(), ov::PropertyMutability::WO}}; + ov::PropertyName{ov::internal::threads_per_stream.name(), ov::PropertyMutability::RW}}; } else if (ov::available_devices == name) { // TODO: fill list of available devices return decltype(ov::available_devices)::value_type{{""}}; diff --git a/src/tests/functional/plugin/shared/include/behavior/compiled_model/compiled_model_base.hpp b/src/tests/functional/plugin/shared/include/behavior/compiled_model/compiled_model_base.hpp index 3445b16878f050..ae6ad2d3324ab8 100644 --- a/src/tests/functional/plugin/shared/include/behavior/compiled_model/compiled_model_base.hpp +++ b/src/tests/functional/plugin/shared/include/behavior/compiled_model/compiled_model_base.hpp @@ -629,7 +629,14 @@ TEST_P(OVCompiledModelBaseTest, canLoadCorrectNetworkToGetExecutableWithIncorrec config.emplace(confItem.first, confItem.second); } - OV_ASSERT_NO_THROW(std::ignore = core->compile_model(function, target_device, config)); + bool is_meta_devices = target_device.find("AUTO") != std::string::npos || + target_device.find("MULTI") != std::string::npos || + target_device.find("HETERO") != std::string::npos; + if (is_meta_devices) { + EXPECT_NO_THROW(auto execNet = core->compile_model(function, target_device, config)); + } else { + EXPECT_ANY_THROW(auto execNet = core->compile_model(function, target_device, config)); + } } typedef std::tuple Date: Tue, 10 Dec 2024 12:59:49 +0000 Subject: [PATCH 16/21] Revert changes Signed-off-by: Raasz, Pawel --- src/inference/src/dev/core_impl.cpp | 7 +++++-- src/plugins/auto_batch/src/plugin.cpp | 12 +++--------- src/plugins/auto_batch/src/plugin.hpp | 3 --- src/plugins/hetero/src/plugin.cpp | 3 +-- src/plugins/intel_cpu/src/plugin.cpp | 3 ++- .../behavior/compiled_model/compiled_model_base.hpp | 1 - 6 files changed, 11 insertions(+), 18 deletions(-) diff --git a/src/inference/src/dev/core_impl.cpp b/src/inference/src/dev/core_impl.cpp index 25dde8973d9356..50d3d9cc2b6610 100644 --- a/src/inference/src/dev/core_impl.cpp +++ b/src/inference/src/dev/core_impl.cpp @@ -714,6 +714,8 @@ ov::Plugin ov::CoreImpl::get_plugin(const std::string& pluginName) const { } } } + + // set global device-id independent settings to plugin plugin.set_property(desc.defaultConfig); }); } @@ -980,6 +982,7 @@ ov::AnyMap ov::CoreImpl::get_supported_property(const std::string& full_device_n if (device_priorities_it != return_properties.end()) { return_properties.erase(device_priorities_it); } + return return_properties; } @@ -998,7 +1001,7 @@ ov::AnyMap ov::CoreImpl::get_supported_property(const std::string& full_device_n // try to search against OV API 2.0' mutable supported_properties try { - for (auto&& property : ICore::get_property(device_name, ov::supported_properties)) { + for (auto&& property : ICore::get_property(device_name, ov::supported_properties, {})) { if (property.is_mutable()) { supported_config_keys.emplace_back(std::move(property)); } @@ -1008,7 +1011,7 @@ ov::AnyMap ov::CoreImpl::get_supported_property(const std::string& full_device_n // try to search against internal supported_properties try { - for (auto&& property : ICore::get_property(device_name, ov::internal::supported_properties)) { + for (auto&& property : ICore::get_property(device_name, ov::internal::supported_properties, {})) { if (property.is_mutable()) { supported_config_keys.emplace_back(std::move(property)); } diff --git a/src/plugins/auto_batch/src/plugin.cpp b/src/plugins/auto_batch/src/plugin.cpp index 3e1b35fe338612..2e3e17cd43086d 100644 --- a/src/plugins/auto_batch/src/plugin.cpp +++ b/src/plugins/auto_batch/src/plugin.cpp @@ -47,16 +47,10 @@ DeviceInformation Plugin::parse_batch_device(const std::string& device_with_batc return {std::move(deviceName), {{}}, static_cast(batch)}; } -DeviceInformation Plugin::parse_only_meta_device(const std::string& devices_batch_config, - const ov::AnyMap& user_config) const { - auto meta_device = parse_batch_device(devices_batch_config); - meta_device.device_config = get_core()->get_supported_property(meta_device.device_name, user_config); - return meta_device; -} - DeviceInformation Plugin::parse_meta_device(const std::string& devices_batch_config, const ov::AnyMap& user_config) const { - auto meta_device = parse_only_meta_device(devices_batch_config, user_config); + auto meta_device = parse_batch_device(devices_batch_config); + meta_device.device_config = get_core()->get_supported_property(meta_device.device_name, user_config); // check that no irrelevant config-keys left for (const auto& k : user_config) { const auto& name = k.first; @@ -146,7 +140,7 @@ std::shared_ptr Plugin::compile_model(const std::shared_ptr< if (device_batch == full_properties.end()) { OPENVINO_THROW("ov::device::priorities key for AUTO BATCH is not set for BATCH device"); } - auto meta_device = parse_only_meta_device(device_batch->second.as(), properties); + auto meta_device = parse_meta_device(device_batch->second.as(), properties); const auto& device_name = meta_device.device_name; const auto& device_config = meta_device.device_config; diff --git a/src/plugins/auto_batch/src/plugin.hpp b/src/plugins/auto_batch/src/plugin.hpp index 8bfdb7c4e8385d..563ba4487ee3ec 100644 --- a/src/plugins/auto_batch/src/plugin.hpp +++ b/src/plugins/auto_batch/src/plugin.hpp @@ -65,9 +65,6 @@ class Plugin : public ov::IPlugin { static DeviceInformation parse_batch_device(const std::string& device_with_batch); private: - DeviceInformation parse_only_meta_device(const std::string& devices_batch_config, - const ov::AnyMap& user_config) const; - mutable ov::AnyMap m_plugin_config; }; } // namespace autobatch_plugin diff --git a/src/plugins/hetero/src/plugin.cpp b/src/plugins/hetero/src/plugin.cpp index 6d2740cfd05001..298be6fa201f4f 100644 --- a/src/plugins/hetero/src/plugin.cpp +++ b/src/plugins/hetero/src/plugin.cpp @@ -283,8 +283,7 @@ ov::Any ov::hetero::Plugin::get_property(const std::string& name, const ov::AnyM return decltype(ov::supported_properties)::value_type(std::move(supported_properties)); } else if (ov::internal::supported_properties == name) { return decltype(ov::internal::supported_properties)::value_type{ - ov::PropertyName{ov::internal::caching_properties.name(), ov::PropertyMutability::RO}, - }; + ov::PropertyName{ov::internal::caching_properties.name(), ov::PropertyMutability::RO}}; } else if (ov::device::full_name == name) { return decltype(ov::device::full_name)::value_type{get_device_name()}; } else if (ov::internal::caching_properties == name) { diff --git a/src/plugins/intel_cpu/src/plugin.cpp b/src/plugins/intel_cpu/src/plugin.cpp index 4e0f9f036e3c8e..6fdbf7a4ea4dee 100644 --- a/src/plugins/intel_cpu/src/plugin.cpp +++ b/src/plugins/intel_cpu/src/plugin.cpp @@ -453,7 +453,8 @@ ov::Any Plugin::get_ro_property(const std::string& name, const ov::AnyMap& optio #endif ov::PropertyName{ov::internal::exclusive_async_requests.name(), ov::PropertyMutability::RW}, ov::PropertyName{ov::internal::compiled_model_runtime_properties.name(), ov::PropertyMutability::RO}, - }; + ov::PropertyName{ov::internal::compiled_model_runtime_properties_supported.name(), + ov::PropertyMutability::RO}}; } else if (name == ov::device::full_name) { return decltype(ov::device::full_name)::value_type(deviceFullName); } else if (name == ov::available_devices) { diff --git a/src/tests/functional/plugin/shared/include/behavior/compiled_model/compiled_model_base.hpp b/src/tests/functional/plugin/shared/include/behavior/compiled_model/compiled_model_base.hpp index ae6ad2d3324ab8..74f6c4193bcf4c 100644 --- a/src/tests/functional/plugin/shared/include/behavior/compiled_model/compiled_model_base.hpp +++ b/src/tests/functional/plugin/shared/include/behavior/compiled_model/compiled_model_base.hpp @@ -628,7 +628,6 @@ TEST_P(OVCompiledModelBaseTest, canLoadCorrectNetworkToGetExecutableWithIncorrec for (const auto& confItem : configuration) { config.emplace(confItem.first, confItem.second); } - bool is_meta_devices = target_device.find("AUTO") != std::string::npos || target_device.find("MULTI") != std::string::npos || target_device.find("HETERO") != std::string::npos; From 434b67f8e30a80c5e5f5401aaaf44e3b1c554d71 Mon Sep 17 00:00:00 2001 From: "Raasz, Pawel" Date: Wed, 11 Dec 2024 05:38:01 +0000 Subject: [PATCH 17/21] Update `parseDeviceNameIntoConfig` to provide local core configuration Signed-off-by: Raasz, Pawel --- src/inference/src/dev/core_impl.cpp | 127 +++++++++++++++++----------- src/inference/src/dev/core_impl.hpp | 109 ++++++++++++++++-------- 2 files changed, 150 insertions(+), 86 deletions(-) diff --git a/src/inference/src/dev/core_impl.cpp b/src/inference/src/dev/core_impl.cpp index 50d3d9cc2b6610..87c89ad038fea5 100644 --- a/src/inference/src/dev/core_impl.cpp +++ b/src/inference/src/dev/core_impl.cpp @@ -218,13 +218,17 @@ void clean_batch_properties(const std::string& deviceName, ov::AnyMap& config, c } } -constexpr bool keep_core = true; - static const auto core_properties_names = ov::util::make_array(ov::cache_dir.name(), ov::enable_mmap.name(), ov::force_tbb_terminate.name()); static const auto auto_batch_properties_names = ov::util::make_array(ov::auto_batch_timeout.name(), ov::hint::allow_auto_batching.name()); + +void remove_core_properties(ov::AnyMap& properties) { + for (const auto& name : core_properties_names) { + properties.erase(name); + } +} } // namespace bool ov::is_config_applicable(const std::string& user_device_name, const std::string& subprop_device_name) { @@ -262,6 +266,13 @@ bool ov::is_config_applicable(const std::string& user_device_name, const std::st ov::Parsed ov::parseDeviceNameIntoConfig(const std::string& deviceName, const AnyMap& config, const bool keep_core_property) { + return parseDeviceNameIntoConfig(deviceName, CoreConfig{}, config, keep_core_property); +} + +ov::Parsed ov::parseDeviceNameIntoConfig(const std::string& deviceName, + const CoreConfig& coreConfig, + const AnyMap& config, + const bool keep_core_property) { // check to the validity of device name auto bracket_pos = deviceName.find(")"); while (bracket_pos != std::string::npos) { @@ -272,9 +283,6 @@ ov::Parsed ov::parseDeviceNameIntoConfig(const std::string& deviceName, bracket_pos = deviceName.find(")", bracket_pos + 1); } - auto updated_config = config; - auto updated_device_name = deviceName; - /** Note: auto-batching is already applied by this time, so the call: * core.compile_model("GPU", ov::device::properties("BATCH", ov::auto_batch_timeout(400))); * is transformed and we have here: @@ -288,8 +296,10 @@ ov::Parsed ov::parseDeviceNameIntoConfig(const std::string& deviceName, * So, if one day, we want to add more options in form of ov::allow_, we need to apply it before * 'flatten_sub_properties' call to have proper behavior */ + ov::Parsed parsed{deviceName, flatten_sub_properties(deviceName, config), coreConfig}; + auto& updated_device_name = parsed._deviceName; + auto& updated_config = parsed._config; - updated_config = flatten_sub_properties(deviceName, updated_config); std::string parsed_device_priority; // try to find ':' to extract name of virtual device @@ -322,14 +332,17 @@ ov::Parsed ov::parseDeviceNameIntoConfig(const std::string& deviceName, } }; + parsed._core_config.set(updated_config); // keep batch property only when called from query_supported_property if (!keep_core_property) { for (const auto& name : auto_batch_properties_names) { clean_batch_properties(updated_device_name, updated_config, name); } + for (const auto& name : {ov::enable_mmap.name(), ov::force_tbb_terminate.name()}) { + updated_config.erase(name); + } } - - return {std::move(updated_device_name), std::move(updated_config)}; + return parsed; } ov::CoreImpl::CoreImpl() { @@ -757,16 +770,13 @@ ov::SoPtr ov::CoreImpl::compile_model(const std::shared_ptr< // if auto-batching is applicable, the below function will patch the device name and config accordingly: auto model = apply_auto_batching(model_, deviceName, config_with_batch); - auto local_core_config = coreConfig; - auto parsed = parseDeviceNameIntoConfig(deviceName, config_with_batch, is_proxy_device(deviceName)); - local_core_config.set_and_update(parsed._config); + auto parsed = parseDeviceNameIntoConfig(deviceName, coreConfig, config_with_batch, is_proxy_device(deviceName)); auto plugin = get_plugin(parsed._deviceName); - ov::SoPtr res; - auto cacheManager = local_core_config.get_cache_config_for_device(plugin)._cacheManager; + auto cacheManager = parsed._core_config.get_cache_config_for_device(plugin, parsed._config)._cacheManager; // Skip caching for proxy plugin. HW plugin will load network from the cache if (cacheManager && device_supports_model_caching(plugin) && !is_proxy_device(plugin)) { - CacheContent cacheContent{cacheManager, local_core_config.get_enable_mmap()}; + CacheContent cacheContent{cacheManager, parsed._core_config.get_enable_mmap()}; cacheContent.blobId = ov::ModelCache::compute_hash(model, create_compile_config(plugin, parsed._config)); std::unique_ptr lock = cacheGuard.get_hash_lock(cacheContent.blobId); res = load_model_from_cache(cacheContent, plugin, parsed._config, ov::SoPtr{}, [&]() { @@ -793,16 +803,13 @@ ov::SoPtr ov::CoreImpl::compile_model(const std::shared_ptr< // if auto-batching is applicable, the below function will patch the device name and config accordingly: auto model = apply_auto_batching(model_, deviceName, config_with_batch); - auto local_core_config = coreConfig; - auto parsed = parseDeviceNameIntoConfig(deviceName, config_with_batch, is_proxy_device(deviceName)); - local_core_config.set_and_update(parsed._config); + auto parsed = parseDeviceNameIntoConfig(deviceName, coreConfig, config_with_batch, is_proxy_device(deviceName)); auto plugin = get_plugin(parsed._deviceName); - ov::SoPtr res; - auto cacheManager = local_core_config.get_cache_config_for_device(plugin)._cacheManager; + auto cacheManager = parsed._core_config.get_cache_config_for_device(plugin, parsed._config)._cacheManager; // Skip caching for proxy plugin. HW plugin will load network from the cache if (cacheManager && device_supports_model_caching(plugin) && !is_proxy_device(plugin)) { - CacheContent cacheContent{cacheManager, local_core_config.get_enable_mmap()}; + CacheContent cacheContent{cacheManager, parsed._core_config.get_enable_mmap()}; cacheContent.blobId = ov::ModelCache::compute_hash(model, create_compile_config(plugin, parsed._config)); std::unique_ptr lock = cacheGuard.get_hash_lock(cacheContent.blobId); res = load_model_from_cache(cacheContent, plugin, parsed._config, context, [&]() { @@ -818,23 +825,22 @@ ov::SoPtr ov::CoreImpl::compile_model(const std::string& mod const std::string& device_name, const ov::AnyMap& config) const { OV_ITT_SCOPE(FIRST_INFERENCE, ov::itt::domains::LoadTime, "Core::compile_model::Path"); - auto parsed = parseDeviceNameIntoConfig(device_name, config, keep_core); - auto local_core_config = coreConfig; + auto parsed = parseDeviceNameIntoConfig(device_name, coreConfig, config); // in case of compile_model(file_name), we need to clear-up core-level properties - local_core_config.set_and_update(parsed._config); auto plugin = get_plugin(parsed._deviceName); - ov::SoPtr compiled_model; - auto cacheManager = local_core_config.get_cache_config_for_device(plugin)._cacheManager; - // Skip caching for proxy plugin. HW plugin will load network from the cache + + auto cacheManager = parsed._core_config.get_cache_config_for_device(plugin, parsed._config)._cacheManager; + if (cacheManager && device_supports_model_caching(plugin) && !is_proxy_device(plugin)) { - CacheContent cacheContent{cacheManager, local_core_config.get_enable_mmap(), model_path}; + // Skip caching for proxy plugin. HW plugin will load network from the cache + CacheContent cacheContent{cacheManager, parsed._core_config.get_enable_mmap(), model_path}; cacheContent.blobId = ov::ModelCache::compute_hash(model_path, create_compile_config(plugin, parsed._config)); std::unique_ptr lock = cacheGuard.get_hash_lock(cacheContent.blobId); compiled_model = load_model_from_cache(cacheContent, plugin, parsed._config, ov::SoPtr{}, [&]() { auto model = - ov::util::read_model(model_path, std::string{}, extensions, local_core_config.get_enable_mmap()); + ov::util::read_model(model_path, std::string{}, extensions, parsed._core_config.get_enable_mmap()); return compile_model_and_cache(plugin, model, parsed._config, {}, cacheContent); }); } else { @@ -848,16 +854,14 @@ ov::SoPtr ov::CoreImpl::compile_model(const std::string& mod const std::string& device_name, const ov::AnyMap& config) const { OV_ITT_SCOPED_TASK(ov::itt::domains::OV, "Core::compile_model::from_memory"); - auto parsed = parseDeviceNameIntoConfig(device_name, config, keep_core); - auto local_core_config = coreConfig; - local_core_config.set_and_update(parsed._config); + auto parsed = parseDeviceNameIntoConfig(device_name, coreConfig, config); auto plugin = get_plugin(parsed._deviceName); - ov::SoPtr compiled_model; - auto cacheManager = local_core_config.get_cache_config_for_device(plugin)._cacheManager; + + auto cacheManager = parsed._core_config.get_cache_config_for_device(plugin, parsed._config)._cacheManager; // Skip caching for proxy plugin. HW plugin will load network from the cache if (cacheManager && device_supports_model_caching(plugin) && !is_proxy_device(plugin)) { - CacheContent cacheContent{cacheManager, local_core_config.get_enable_mmap()}; + CacheContent cacheContent{cacheManager, parsed._core_config.get_enable_mmap()}; cacheContent.blobId = ov::ModelCache::compute_hash(model_str, weights, create_compile_config(plugin, parsed._config)); std::unique_ptr lock = cacheGuard.get_hash_lock(cacheContent.blobId); @@ -993,17 +997,17 @@ ov::AnyMap ov::CoreImpl::get_supported_property(const std::string& full_device_n // virtual plugins should bypass core-level properties to HW plugins // so, we need to report them as supported std::vector supported_config_keys; - if (keep_core) { - auto last = supported_config_keys.end(); - last = supported_config_keys.insert(last, core_properties_names.begin(), core_properties_names.end()); - supported_config_keys.insert(last, auto_batch_properties_names.begin(), auto_batch_properties_names.end()); + auto key_inserter = std::back_inserter(supported_config_keys); + if (keep_core_property) { + key_inserter = std::copy(core_properties_names.begin(), core_properties_names.end(), key_inserter); + key_inserter = std::copy(auto_batch_properties_names.begin(), auto_batch_properties_names.end(), key_inserter); } // try to search against OV API 2.0' mutable supported_properties try { for (auto&& property : ICore::get_property(device_name, ov::supported_properties, {})) { if (property.is_mutable()) { - supported_config_keys.emplace_back(std::move(property)); + *key_inserter = std::move(property); } } } catch (ov::Exception&) { @@ -1013,7 +1017,7 @@ ov::AnyMap ov::CoreImpl::get_supported_property(const std::string& full_device_n try { for (auto&& property : ICore::get_property(device_name, ov::internal::supported_properties, {})) { if (property.is_mutable()) { - supported_config_keys.emplace_back(std::move(property)); + *key_inserter = std::move(property); } } } catch (ov::Exception&) { @@ -1533,7 +1537,7 @@ ov::AnyMap ov::CoreImpl::create_compile_config(const ov::Plugin& plugin, const o return compile_config; } -ov::CoreImpl::CoreConfig::CoreConfig(const CoreConfig& other) { +ov::CoreConfig::CoreConfig(const CoreConfig& other) { { std::lock_guard lock(other._cacheConfigMutex); _cacheConfig = other._cacheConfig; @@ -1542,7 +1546,7 @@ ov::CoreImpl::CoreConfig::CoreConfig(const CoreConfig& other) { _flag_enable_mmap = other._flag_enable_mmap; } -void ov::CoreImpl::CoreConfig::set_and_update(ov::AnyMap& config) { +void ov::CoreConfig::set(const ov::AnyMap& config) { auto it = config.find(ov::cache_dir.name()); if (it != config.end()) { std::lock_guard lock(_cacheConfigMutex); @@ -1552,45 +1556,66 @@ void ov::CoreImpl::CoreConfig::set_and_update(ov::AnyMap& config) { for (auto& deviceCfg : _cacheConfigPerDevice) { deviceCfg.second = CoreConfig::CacheConfig::create(it->second.as()); } - config.erase(it); } it = config.find(ov::force_tbb_terminate.name()); if (it != config.end()) { auto flag = it->second.as(); ov::threading::executor_manager()->set_property({{it->first, flag}}); - config.erase(it); } it = config.find(ov::enable_mmap.name()); if (it != config.end()) { auto flag = it->second.as(); _flag_enable_mmap = flag; - config.erase(it); } } -void ov::CoreImpl::CoreConfig::set_cache_dir_for_device(const std::string& dir, const std::string& name) { +void ov::CoreConfig::set_and_update(ov::AnyMap& config) { + set(config); + remove_core_properties(config); +} + +void ov::CoreConfig::set_cache_dir_for_device(const std::string& dir, const std::string& name) { std::lock_guard lock(_cacheConfigMutex); _cacheConfigPerDevice[name] = CoreConfig::CacheConfig::create(dir); } -std::string ov::CoreImpl::CoreConfig::get_cache_dir() const { +std::string ov::CoreConfig::get_cache_dir() const { std::lock_guard lock(_cacheConfigMutex); return _cacheConfig._cacheDir; } -bool ov::CoreImpl::CoreConfig::get_enable_mmap() const { +bool ov::CoreConfig::get_enable_mmap() const { return _flag_enable_mmap; } -ov::CoreImpl::CoreConfig::CacheConfig ov::CoreImpl::CoreConfig::get_cache_config_for_device( - const ov::Plugin& plugin) const { +// Creating thread-safe copy of config including shared_ptr to ICacheManager +// Passing empty or not-existing name will return global cache config +ov::CoreConfig::CacheConfig ov::CoreConfig::get_cache_config_for_device(const ov::Plugin& plugin, + ov::AnyMap& parsedConfig) const { + // cache_dir is enabled locally in compile_model only + if (parsedConfig.count(ov::cache_dir.name())) { + const auto& cache_dir_val = parsedConfig.at(ov::cache_dir.name()).as(); + const auto& tempConfig = CoreConfig::CacheConfig::create(cache_dir_val); + // if plugin does not explicitly support cache_dir, and if plugin is not virtual, we need to remove + // it from config + if (!util::contains(plugin.get_property(ov::supported_properties), ov::cache_dir) && + !is_virtual_device(plugin.get_name())) { + parsedConfig.erase(ov::cache_dir.name()); + } + return tempConfig; + } else { // cache_dir is set to Core globally or for the specific device + return get_cache_config_for_device(plugin); + } +} + +ov::CoreConfig::CacheConfig ov::CoreConfig::get_cache_config_for_device(const ov::Plugin& plugin) const { std::lock_guard lock(_cacheConfigMutex); return _cacheConfigPerDevice.count(plugin.get_name()) ? _cacheConfigPerDevice.at(plugin.get_name()) : _cacheConfig; } -ov::CoreImpl::CoreConfig::CacheConfig ov::CoreImpl::CoreConfig::CacheConfig::create(const std::string& dir) { +ov::CoreConfig::CacheConfig ov::CoreConfig::CacheConfig::create(const std::string& dir) { std::shared_ptr cache_manager = nullptr; if (!dir.empty()) { diff --git a/src/inference/src/dev/core_impl.hpp b/src/inference/src/dev/core_impl.hpp index ceaecb53c296de..181a9235062714 100644 --- a/src/inference/src/dev/core_impl.hpp +++ b/src/inference/src/dev/core_impl.hpp @@ -22,12 +22,86 @@ using CreatePluginEngineFunc = void(std::shared_ptr<::ov::IPlugin>&); const std::string DEFAULT_DEVICE_NAME = "DEFAULT_DEVICE"; +class CoreConfig final { +public: + CoreConfig() = default; + CoreConfig(const CoreConfig& other); + + struct CacheConfig { + std::string _cacheDir; + std::shared_ptr _cacheManager; + + static CacheConfig create(const std::string& dir); + }; + + void set(const ov::AnyMap& config); + + /** + * @brief Removes core-level properties from config and triggers new state for core config + * @param config - config to be updated + */ + void set_and_update(ov::AnyMap& config); + + OPENVINO_DEPRECATED("Don't use this method, it will be removed soon") + void set_cache_dir_for_device(const std::string& dir, const std::string& name); + + std::string get_cache_dir() const; + + bool get_enable_mmap() const; + + CacheConfig get_cache_config_for_device(const ov::Plugin& plugin, ov::AnyMap& parsedConfig) const; + + // Creating thread-safe copy of global config including shared_ptr to ICacheManager + CacheConfig get_cache_config_for_device(const ov::Plugin& plugin) const; + +private: + mutable std::mutex _cacheConfigMutex; + CacheConfig _cacheConfig; + std::map _cacheConfigPerDevice; + bool _flag_enable_mmap = true; +}; + struct Parsed { std::string _deviceName; AnyMap _config; + CoreConfig _core_config; }; +/** + * @brief Provides Parsed device name and configuration. + * + * Uses default core configuration updated with user properties from config. + * @note The `CACHE_DIR` is not removed from compiled configuration. + * + * @param deviceName Device name to be parsed + * @param config User configuration to be parsed. + * @param keep_core_property If set keep core properties in compile properties. + * @return Parsed: + * - device name + * - compile properties + * - core configuration + */ +Parsed parseDeviceNameIntoConfig(const std::string& deviceName, + const AnyMap& config = {}, + const bool keep_core_property = false); + +/** + * @brief Provides Parsed device name and configuration. + * + * Uses user core configuration which is updated with user properties from config. + * @note The `CACHE_DIR` is not removed from compiled configuration. + * + * @param deviceName Device name to be parsed + * @param coreConfig Core configuration used as base for parsed output. + * @param config User configuration to be parsed. + * @param keep_core_property If set keep core properties in compile properties. + * @return Parsed: + * - device name + * - compile properties + * - core configuration + */ Parsed parseDeviceNameIntoConfig(const std::string& deviceName, + const CoreConfig& coreConfig, const AnyMap& config = {}, const bool keep_core_property = false); @@ -61,41 +135,6 @@ class CoreImpl : public ov::ICore, public std::enable_shared_from_this _cacheManager; - - static CacheConfig create(const std::string& dir); - }; - - /** - * @brief Removes core-level properties from config and triggers new state for core config - * @param config - config to be updated - */ - void set_and_update(ov::AnyMap& config); - - OPENVINO_DEPRECATED("Don't use this method, it will be removed soon") - void set_cache_dir_for_device(const std::string& dir, const std::string& name); - - std::string get_cache_dir() const; - - bool get_enable_mmap() const; - - // Creating thread-safe copy of global config including shared_ptr to ICacheManager - CacheConfig get_cache_config_for_device(const ov::Plugin& plugin) const; - - private: - mutable std::mutex _cacheConfigMutex; - CacheConfig _cacheConfig; - std::map _cacheConfigPerDevice; - bool _flag_enable_mmap = true; - }; - struct CacheContent { explicit CacheContent(const std::shared_ptr& cache_manager, bool mmap_enabled = false, From c0b700bb75e81799856baa97205c57540775fbf4 Mon Sep 17 00:00:00 2001 From: "Raasz, Pawel" Date: Wed, 11 Dec 2024 06:01:10 +0000 Subject: [PATCH 18/21] Fix issues after merge Signed-off-by: Raasz, Pawel --- src/inference/src/dev/core_impl.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/inference/src/dev/core_impl.cpp b/src/inference/src/dev/core_impl.cpp index 1e7ba0947b1fbb..7cf3050b741bbb 100644 --- a/src/inference/src/dev/core_impl.cpp +++ b/src/inference/src/dev/core_impl.cpp @@ -1437,7 +1437,7 @@ ov::SoPtr ov::CoreImpl::load_model_from_cache( cacheContent.blobId, cacheContent.mmap_enabled && ov::util::contains(plugin.get_property(ov::internal::supported_properties), ov::internal::caching_with_mmap), - [&](std::istream& networkStream, , std::shared_ptr model_buffer) { + [&](std::istream& networkStream, std::shared_ptr model_buffer) { OV_ITT_SCOPE(FIRST_INFERENCE, ov::itt::domains::LoadTime, "Core::load_model_from_cache::ReadStreamAndImport"); From 7dc7b73b2ab4d466b1986fa54856217f40743595 Mon Sep 17 00:00:00 2001 From: "Raasz, Pawel" Date: Wed, 11 Dec 2024 09:44:23 +0000 Subject: [PATCH 19/21] Clean core properties for HW devices in parseDeviceNameIntoConfig Signed-off-by: Raasz, Pawel --- src/inference/src/dev/core_impl.cpp | 127 +++++++++++++++------------- src/inference/src/dev/core_impl.hpp | 20 +++-- 2 files changed, 81 insertions(+), 66 deletions(-) diff --git a/src/inference/src/dev/core_impl.cpp b/src/inference/src/dev/core_impl.cpp index 7cf3050b741bbb..df0cedad955e4c 100644 --- a/src/inference/src/dev/core_impl.cpp +++ b/src/inference/src/dev/core_impl.cpp @@ -229,58 +229,19 @@ void remove_core_properties(ov::AnyMap& properties) { properties.erase(name); } } -} // namespace - -bool ov::is_config_applicable(const std::string& user_device_name, const std::string& subprop_device_name) { - // full match - if (user_device_name == subprop_device_name) - return true; - - auto parsed_user_device_name = ov::parseDeviceNameIntoConfig(user_device_name); - auto parsed_subprop_device_name = ov::parseDeviceNameIntoConfig(subprop_device_name); - - // if device name is matched, check additional condition - auto is_matched = [&](const std::string& key, MatchType match_type) -> bool { - const auto& user_value = - parsed_user_device_name._config.count(key) ? parsed_user_device_name._config.at(key).as() : ""; - const auto& subprop_value = parsed_subprop_device_name._config.count(key) - ? parsed_subprop_device_name._config.at(key).as() - : ""; - if (!user_value.empty() && subprop_value.empty()) { - // property without additional limitation can be applied - return true; - } - return match_type == MatchType::EXACT ? (user_value == subprop_value) : (user_value.find(subprop_value) == 0); - return false; - }; - - if (parsed_user_device_name._deviceName == parsed_subprop_device_name._deviceName) { - auto device_priority = get_device_priority_property(parsed_user_device_name._deviceName); - return is_matched(device_priority.prop_name, device_priority.match_type); - } - - return false; -} - -ov::Parsed ov::parseDeviceNameIntoConfig(const std::string& deviceName, - const AnyMap& config, - const bool keep_core_property) { - return parseDeviceNameIntoConfig(deviceName, CoreConfig{}, config, keep_core_property); -} - -ov::Parsed ov::parseDeviceNameIntoConfig(const std::string& deviceName, - const CoreConfig& coreConfig, - const AnyMap& config, - const bool keep_core_property) { +ov::Parsed parse_device_config(const std::string& device_name, + const ov::CoreConfig& core_config, + const ov::AnyMap& properties, + const bool keep_auto_batch_property) { // check to the validity of device name - auto bracket_pos = deviceName.find(")"); + auto bracket_pos = device_name.find(")"); while (bracket_pos != std::string::npos) { - if (bracket_pos < deviceName.length() - 1 && - (deviceName[bracket_pos + 1] != ',' || bracket_pos + 1 == deviceName.length() - 1)) { - OPENVINO_THROW("Device with \"", deviceName, "\" name is illegal in the OpenVINO Runtime"); + if (bracket_pos < device_name.length() - 1 && + (device_name[bracket_pos + 1] != ',' || bracket_pos + 1 == device_name.length() - 1)) { + OPENVINO_THROW("Device with \"", device_name, "\" name is illegal in the OpenVINO Runtime"); } - bracket_pos = deviceName.find(")", bracket_pos + 1); + bracket_pos = device_name.find(")", bracket_pos + 1); } /** Note: auto-batching is already applied by this time, so the call: @@ -296,19 +257,19 @@ ov::Parsed ov::parseDeviceNameIntoConfig(const std::string& deviceName, * So, if one day, we want to add more options in form of ov::allow_, we need to apply it before * 'flatten_sub_properties' call to have proper behavior */ - ov::Parsed parsed{deviceName, flatten_sub_properties(deviceName, config), coreConfig}; + ov::Parsed parsed{device_name, flatten_sub_properties(device_name, properties), core_config}; auto& updated_device_name = parsed._deviceName; auto& updated_config = parsed._config; std::string parsed_device_priority; // try to find ':' to extract name of virtual device - auto pos = deviceName.find_first_of(':'); + auto pos = device_name.find_first_of(':'); if (pos != std::string::npos) { - updated_device_name = deviceName.substr(0, pos); - parsed_device_priority = deviceName.substr(pos + 1); + updated_device_name = device_name.substr(0, pos); + parsed_device_priority = device_name.substr(pos + 1); } else { - ov::DeviceIDParser parser(deviceName); + ov::DeviceIDParser parser(device_name); updated_device_name = parser.get_device_name(); parsed_device_priority = parser.get_device_id(); } @@ -325,7 +286,7 @@ ov::Parsed ov::parseDeviceNameIntoConfig(const std::string& deviceName, OPENVINO_THROW("Device priority / ID mismatch: ", parsed_device_priority, " (from ", - deviceName, + device_name, ") vs ", it->second.as(), " (from config)"); @@ -334,12 +295,64 @@ ov::Parsed ov::parseDeviceNameIntoConfig(const std::string& deviceName, parsed._core_config.set(updated_config); // keep batch property only when called from query_supported_property - if (!keep_core_property) { + if (!keep_auto_batch_property) { for (const auto& name : auto_batch_properties_names) { clean_batch_properties(updated_device_name, updated_config, name); } + } + return parsed; +} +} // namespace + +bool ov::is_config_applicable(const std::string& user_device_name, const std::string& subprop_device_name) { + // full match + if (user_device_name == subprop_device_name) + return true; + + auto parsed_user_device_name = ov::parseDeviceNameIntoConfig(user_device_name); + auto parsed_subprop_device_name = ov::parseDeviceNameIntoConfig(subprop_device_name); + + // if device name is matched, check additional condition + auto is_matched = [&](const std::string& key, MatchType match_type) -> bool { + const auto& user_value = + parsed_user_device_name._config.count(key) ? parsed_user_device_name._config.at(key).as() : ""; + const auto& subprop_value = parsed_subprop_device_name._config.count(key) + ? parsed_subprop_device_name._config.at(key).as() + : ""; + + if (!user_value.empty() && subprop_value.empty()) { + // property without additional limitation can be applied + return true; + } + return match_type == MatchType::EXACT ? (user_value == subprop_value) : (user_value.find(subprop_value) == 0); + return false; + }; + + if (parsed_user_device_name._deviceName == parsed_subprop_device_name._deviceName) { + auto device_priority = get_device_priority_property(parsed_user_device_name._deviceName); + return is_matched(device_priority.prop_name, device_priority.match_type); + } + + return false; +} + +ov::Parsed ov::parseDeviceNameIntoConfig(const std::string& deviceName, + const AnyMap& config, + const bool keep_auto_batch_property) { + return parseDeviceNameIntoConfig(deviceName, CoreConfig{}, config, keep_auto_batch_property); +} + +ov::Parsed ov::parseDeviceNameIntoConfig(const std::string& deviceName, + const CoreConfig& coreConfig, + const AnyMap& config, + const bool keep_auto_batch_property) { + auto parsed = parse_device_config(deviceName, coreConfig, config, keep_auto_batch_property); + + // remove core properties for HW devices + if (!is_virtual_device(parsed._deviceName)) { for (const auto& name : {ov::enable_mmap.name(), ov::force_tbb_terminate.name()}) { - updated_config.erase(name); + // note: ov::cache_dir kept as plugin may require it + parsed._config.erase(name); } } return parsed; @@ -990,7 +1003,7 @@ ov::AnyMap ov::CoreImpl::get_supported_property(const std::string& full_device_n return return_properties; } - const auto flattened = parseDeviceNameIntoConfig(full_device_name, user_properties, keep_core_property); + const auto flattened = parse_device_config(full_device_name, {}, user_properties, keep_core_property); const auto& flattened_config = flattened._config; const auto& device_name = flattened._deviceName; diff --git a/src/inference/src/dev/core_impl.hpp b/src/inference/src/dev/core_impl.hpp index 181a9235062714..2d13d56cd9a80c 100644 --- a/src/inference/src/dev/core_impl.hpp +++ b/src/inference/src/dev/core_impl.hpp @@ -71,11 +71,12 @@ struct Parsed { * @brief Provides Parsed device name and configuration. * * Uses default core configuration updated with user properties from config. + * The core properties are removed from user configuration for HW devices only. * @note The `CACHE_DIR` is not removed from compiled configuration. * - * @param deviceName Device name to be parsed - * @param config User configuration to be parsed. - * @param keep_core_property If set keep core properties in compile properties. + * @param deviceName Device name to be parsed + * @param config User configuration to be parsed. + * @param keep_auto_batch_property If set keep auto batch properties in compile properties. * @return Parsed: * - device name * - compile properties @@ -83,18 +84,19 @@ struct Parsed { */ Parsed parseDeviceNameIntoConfig(const std::string& deviceName, const AnyMap& config = {}, - const bool keep_core_property = false); + const bool keep_auto_batch_property = false); /** * @brief Provides Parsed device name and configuration. * * Uses user core configuration which is updated with user properties from config. + * The core properties are removed from user configuration for HW devices only. * @note The `CACHE_DIR` is not removed from compiled configuration. * - * @param deviceName Device name to be parsed - * @param coreConfig Core configuration used as base for parsed output. - * @param config User configuration to be parsed. - * @param keep_core_property If set keep core properties in compile properties. + * @param deviceName Device name to be parsed + * @param coreConfig Core configuration used as base for parsed output. + * @param config User configuration to be parsed. + * @param keep_auto_batch_property If set keep auto batch properties in compile properties. * @return Parsed: * - device name * - compile properties @@ -103,7 +105,7 @@ Parsed parseDeviceNameIntoConfig(const std::string& deviceName, Parsed parseDeviceNameIntoConfig(const std::string& deviceName, const CoreConfig& coreConfig, const AnyMap& config = {}, - const bool keep_core_property = false); + const bool keep_auto_batch_property = false); /** * @brief Checks whether config is applicable for device with 'device_name' From 9c7f13a1c41c01bc415d559f5adcefe3cafea0e2 Mon Sep 17 00:00:00 2001 From: "Raasz, Pawel" Date: Wed, 11 Dec 2024 09:47:32 +0000 Subject: [PATCH 20/21] Add comment about cache_dir removal from compile config Signed-off-by: Raasz, Pawel --- src/inference/src/dev/core_impl.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/inference/src/dev/core_impl.cpp b/src/inference/src/dev/core_impl.cpp index df0cedad955e4c..c192c3b67c6c65 100644 --- a/src/inference/src/dev/core_impl.cpp +++ b/src/inference/src/dev/core_impl.cpp @@ -786,6 +786,7 @@ ov::SoPtr ov::CoreImpl::compile_model(const std::shared_ptr< auto parsed = parseDeviceNameIntoConfig(deviceName, coreConfig, config_with_batch, is_proxy_device(deviceName)); auto plugin = get_plugin(parsed._deviceName); ov::SoPtr res; + // will consume ov::cache_dir if plugin not support it auto cacheManager = parsed._core_config.get_cache_config_for_device(plugin, parsed._config)._cacheManager; // Skip caching for proxy plugin. HW plugin will load network from the cache if (cacheManager && device_supports_model_caching(plugin) && !is_proxy_device(plugin)) { @@ -819,6 +820,7 @@ ov::SoPtr ov::CoreImpl::compile_model(const std::shared_ptr< auto parsed = parseDeviceNameIntoConfig(deviceName, coreConfig, config_with_batch, is_proxy_device(deviceName)); auto plugin = get_plugin(parsed._deviceName); ov::SoPtr res; + // will consume ov::cache_dir if plugin not support it auto cacheManager = parsed._core_config.get_cache_config_for_device(plugin, parsed._config)._cacheManager; // Skip caching for proxy plugin. HW plugin will load network from the cache if (cacheManager && device_supports_model_caching(plugin) && !is_proxy_device(plugin)) { @@ -842,7 +844,7 @@ ov::SoPtr ov::CoreImpl::compile_model(const std::string& mod // in case of compile_model(file_name), we need to clear-up core-level properties auto plugin = get_plugin(parsed._deviceName); ov::SoPtr compiled_model; - + // will consume ov::cache_dir if plugin not support it auto cacheManager = parsed._core_config.get_cache_config_for_device(plugin, parsed._config)._cacheManager; if (cacheManager && device_supports_model_caching(plugin) && !is_proxy_device(plugin)) { @@ -870,7 +872,7 @@ ov::SoPtr ov::CoreImpl::compile_model(const std::string& mod auto parsed = parseDeviceNameIntoConfig(device_name, coreConfig, config); auto plugin = get_plugin(parsed._deviceName); ov::SoPtr compiled_model; - + // will consume ov::cache_dir if plugin not support it auto cacheManager = parsed._core_config.get_cache_config_for_device(plugin, parsed._config)._cacheManager; // Skip caching for proxy plugin. HW plugin will load network from the cache if (cacheManager && device_supports_model_caching(plugin) && !is_proxy_device(plugin)) { From 511bb880a8a198c4b51b66e1eddc4b39796ef4c3 Mon Sep 17 00:00:00 2001 From: "Raasz, Pawel" Date: Wed, 11 Dec 2024 10:49:46 +0000 Subject: [PATCH 21/21] Re order code in core_impl.cpp Signed-off-by: Raasz, Pawel --- src/inference/src/dev/core_impl.cpp | 66 +++++++++++++++-------------- 1 file changed, 34 insertions(+), 32 deletions(-) diff --git a/src/inference/src/dev/core_impl.cpp b/src/inference/src/dev/core_impl.cpp index c192c3b67c6c65..f332c7c999a548 100644 --- a/src/inference/src/dev/core_impl.cpp +++ b/src/inference/src/dev/core_impl.cpp @@ -229,7 +229,41 @@ void remove_core_properties(ov::AnyMap& properties) { properties.erase(name); } } +} // namespace + +bool ov::is_config_applicable(const std::string& user_device_name, const std::string& subprop_device_name) { + // full match + if (user_device_name == subprop_device_name) + return true; + + auto parsed_user_device_name = ov::parseDeviceNameIntoConfig(user_device_name); + auto parsed_subprop_device_name = ov::parseDeviceNameIntoConfig(subprop_device_name); + + // if device name is matched, check additional condition + auto is_matched = [&](const std::string& key, MatchType match_type) -> bool { + const auto& user_value = + parsed_user_device_name._config.count(key) ? parsed_user_device_name._config.at(key).as() : ""; + const auto& subprop_value = parsed_subprop_device_name._config.count(key) + ? parsed_subprop_device_name._config.at(key).as() + : ""; + + if (!user_value.empty() && subprop_value.empty()) { + // property without additional limitation can be applied + return true; + } + return match_type == MatchType::EXACT ? (user_value == subprop_value) : (user_value.find(subprop_value) == 0); + return false; + }; + + if (parsed_user_device_name._deviceName == parsed_subprop_device_name._deviceName) { + auto device_priority = get_device_priority_property(parsed_user_device_name._deviceName); + return is_matched(device_priority.prop_name, device_priority.match_type); + } + + return false; +} +namespace { ov::Parsed parse_device_config(const std::string& device_name, const ov::CoreConfig& core_config, const ov::AnyMap& properties, @@ -304,38 +338,6 @@ ov::Parsed parse_device_config(const std::string& device_name, } } // namespace -bool ov::is_config_applicable(const std::string& user_device_name, const std::string& subprop_device_name) { - // full match - if (user_device_name == subprop_device_name) - return true; - - auto parsed_user_device_name = ov::parseDeviceNameIntoConfig(user_device_name); - auto parsed_subprop_device_name = ov::parseDeviceNameIntoConfig(subprop_device_name); - - // if device name is matched, check additional condition - auto is_matched = [&](const std::string& key, MatchType match_type) -> bool { - const auto& user_value = - parsed_user_device_name._config.count(key) ? parsed_user_device_name._config.at(key).as() : ""; - const auto& subprop_value = parsed_subprop_device_name._config.count(key) - ? parsed_subprop_device_name._config.at(key).as() - : ""; - - if (!user_value.empty() && subprop_value.empty()) { - // property without additional limitation can be applied - return true; - } - return match_type == MatchType::EXACT ? (user_value == subprop_value) : (user_value.find(subprop_value) == 0); - return false; - }; - - if (parsed_user_device_name._deviceName == parsed_subprop_device_name._deviceName) { - auto device_priority = get_device_priority_property(parsed_user_device_name._deviceName); - return is_matched(device_priority.prop_name, device_priority.match_type); - } - - return false; -} - ov::Parsed ov::parseDeviceNameIntoConfig(const std::string& deviceName, const AnyMap& config, const bool keep_auto_batch_property) {