[GPU] Parse runtime_options from model RT info and apply to config (#…

…27896)

Signed-off-by: Vladimir Paramuzov <[email protected]>

src/plugins/intel_gpu/include/intel_gpu/runtime/execution_config.hpp

@@ -138,6 +138,10 @@ class ExecutionConfig {
 
     void apply_user_properties(const cldnn::device_info& info);
 
+    // Note that RT info property value has lower priority than values set by user via core.set_property or passed to compile_model call
+    // So this method should be called after setting all user properties, but before apply_user_properties() call.
+    void apply_rt_info(const ov::RTMap& rt_info);
+
     std::string to_string() const;
 
 protected:
@@ -147,6 +151,16 @@ class ExecutionConfig {
     void apply_priority_hints(const cldnn::device_info& info);
     void apply_debug_options(const cldnn::device_info& info);
 
+    template <typename T, PropertyMutability mutability>
+    void apply_rt_info_property(const ov::Property<T, mutability>& property, const ov::RTMap& rt_info) {
+        if (!is_set_by_user(property)) {
+            auto rt_info_val = rt_info.find(property.name());
+            if (rt_info_val != rt_info.end()) {
+                set_user_property(property(rt_info_val->second.template as<T>()));
+            }
+        }
+    }
+
 private:
     ov::AnyMap internal_properties;
     ov::AnyMap user_properties;

src/plugins/intel_gpu/src/plugin/plugin.cpp

@@ -189,6 +189,8 @@ std::shared_ptr<ov::ICompiledModel> Plugin::compile_model(const std::shared_ptr<
 
     ExecutionConfig config = m_configs_map.at(device_id);
     config.set_user_property(orig_config);
+    if (model->has_rt_info("runtime_options"))
+        config.apply_rt_info(model->get_rt_info<ov::AnyMap>("runtime_options"));
     config.apply_user_properties(context->get_engine().get_device_info());
 
     set_cache_info(model, config);
@@ -278,6 +280,8 @@ ov::SupportedOpsMap Plugin::query_model(const std::shared_ptr<const ov::Model>&
 
     ExecutionConfig config = m_configs_map.at(device_id);
     config.set_user_property(orig_config);
+    if (model->has_rt_info("runtime_options"))
+        config.apply_rt_info(model->get_rt_info<ov::AnyMap>("runtime_options"));
     config.apply_user_properties(ctx->get_engine().get_device_info());
 
     ProgramBuilder prog(ctx->get_engine(), config);

src/plugins/intel_gpu/src/runtime/execution_config.cpp

@@ -254,6 +254,12 @@ void ExecutionConfig::apply_user_properties(const cldnn::device_info& info) {
     user_properties.clear();
 }
 
+void ExecutionConfig::apply_rt_info(const ov::RTMap& rt_info) {
+    apply_rt_info_property(ov::hint::kv_cache_precision, rt_info);
+    apply_rt_info_property(ov::hint::dynamic_quantization_group_size, rt_info);
+    apply_rt_info_property(ov::hint::activations_scale_factor, rt_info);
+}
+
 std::string ExecutionConfig::to_string() const {
     std::stringstream s;
     s << "internal properties:\n";

src/plugins/intel_gpu/tests/functional/behavior/properties.cpp

@@ -0,0 +1,99 @@
+// Copyright (C) 2024 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+
+#include "openvino/runtime/properties.hpp"
+#include "base/ov_behavior_test_utils.hpp"
+#include "openvino/runtime/core.hpp"
+#include "common_test_utils/subgraph_builders/conv_pool_relu.hpp"
+
+namespace {
+
+class TestPropertiesGPU : public ::testing::Test {
+public:
+    std::shared_ptr<ov::Model> model;
+
+    void SetUp() override {
+        SKIP_IF_CURRENT_TEST_IS_DISABLED();
+        model = ov::test::utils::make_conv_pool_relu();
+    }
+};
+
+TEST_F(TestPropertiesGPU, NoRTInfo) {
+    ov::Core core;
+    ov::Any type;
+    ov::Any size;
+    ov::Any scale;
+    ov::CompiledModel compiled_model;
+
+    OV_ASSERT_NO_THROW(compiled_model = core.compile_model(model, ov::test::utils::DEVICE_GPU));
+    OV_ASSERT_NO_THROW(type = compiled_model.get_property(ov::hint::kv_cache_precision));
+    OV_ASSERT_NO_THROW(size = compiled_model.get_property(ov::hint::dynamic_quantization_group_size));
+    OV_ASSERT_NO_THROW(scale = compiled_model.get_property(ov::hint::activations_scale_factor));
+}
+
+TEST_F(TestPropertiesGPU, RTInfoPropertiesWithDefault) {
+    ov::Core core;
+    ov::Any type;
+    ov::Any size;
+    ov::Any scale;
+    ov::CompiledModel compiled_model;
+    model->set_rt_info("f16", "runtime_options", ov::hint::kv_cache_precision.name());
+    model->set_rt_info("0", "runtime_options", ov::hint::dynamic_quantization_group_size.name());
+    model->set_rt_info("8.0", "runtime_options", ov::hint::activations_scale_factor.name());
+
+    OV_ASSERT_NO_THROW(compiled_model = core.compile_model(model, ov::test::utils::DEVICE_GPU));
+    OV_ASSERT_NO_THROW(type = compiled_model.get_property(ov::hint::kv_cache_precision));
+    OV_ASSERT_NO_THROW(size = compiled_model.get_property(ov::hint::dynamic_quantization_group_size));
+    OV_ASSERT_NO_THROW(scale = compiled_model.get_property(ov::hint::activations_scale_factor));
+    ASSERT_EQ(type.as<ov::element::Type>(), ov::element::f16);
+    ASSERT_EQ(size.as<uint64_t>(), 0);
+    ASSERT_EQ(scale.as<float>(), 8.0f);
+}
+
+TEST_F(TestPropertiesGPU, RTInfoPropertiesWithUserValuesFromCore) {
+    ov::Core core;
+    ov::Any type;
+    ov::Any size;
+    ov::Any scale;
+    ov::CompiledModel compiled_model;
+    model->set_rt_info("f16", "runtime_options", ov::hint::kv_cache_precision.name());
+    model->set_rt_info("0", "runtime_options", ov::hint::dynamic_quantization_group_size.name());
+    model->set_rt_info("8.0", "runtime_options", ov::hint::activations_scale_factor.name());
+    core.set_property(ov::hint::kv_cache_precision(ov::element::u8));
+    core.set_property(ov::hint::dynamic_quantization_group_size(16));
+    core.set_property(ov::hint::activations_scale_factor(4.0f));
+
+    OV_ASSERT_NO_THROW(compiled_model = core.compile_model(model, ov::test::utils::DEVICE_GPU));
+    OV_ASSERT_NO_THROW(type = compiled_model.get_property(ov::hint::kv_cache_precision));
+    OV_ASSERT_NO_THROW(size = compiled_model.get_property(ov::hint::dynamic_quantization_group_size));
+    OV_ASSERT_NO_THROW(scale = compiled_model.get_property(ov::hint::activations_scale_factor));
+    ASSERT_EQ(type.as<ov::element::Type>(), ov::element::u8);
+    ASSERT_EQ(size.as<uint64_t>(), 16);
+    ASSERT_EQ(scale.as<float>(), 4.0f);
+}
+
+TEST_F(TestPropertiesGPU, RTInfoPropertiesWithUserValuesFromCompileModel) {
+    ov::Core core;
+    ov::Any type;
+    ov::Any size;
+    ov::Any scale;
+    ov::CompiledModel compiled_model;
+    model->set_rt_info("f16", "runtime_options", ov::hint::kv_cache_precision.name());
+    model->set_rt_info("0", "runtime_options", ov::hint::dynamic_quantization_group_size.name());
+    model->set_rt_info("8.0", "runtime_options", ov::hint::activations_scale_factor.name());
+    ov::AnyMap config;
+    config[ov::hint::kv_cache_precision.name()] = "u8";
+    config[ov::hint::dynamic_quantization_group_size.name()] = "16";
+    config[ov::hint::activations_scale_factor.name()] = "4.0";
+
+    OV_ASSERT_NO_THROW(compiled_model = core.compile_model(model, ov::test::utils::DEVICE_GPU, config));
+    OV_ASSERT_NO_THROW(type = compiled_model.get_property(ov::hint::kv_cache_precision));
+    OV_ASSERT_NO_THROW(size = compiled_model.get_property(ov::hint::dynamic_quantization_group_size));
+    OV_ASSERT_NO_THROW(scale = compiled_model.get_property(ov::hint::activations_scale_factor));
+    ASSERT_EQ(type.as<ov::element::Type>(), ov::element::u8);
+    ASSERT_EQ(size.as<uint64_t>(), 16);
+    ASSERT_EQ(scale.as<float>(), 4.0f);
+}
+
+} // namespace