From 71a0bb87fc2397800a4e0095a6a2d7dad388af93 Mon Sep 17 00:00:00 2001 From: eaidova Date: Thu, 24 Oct 2024 15:18:24 +0400 Subject: [PATCH 1/4] fix conversion for text embeddings for fp16 models --- optimum/exporters/openvino/convert.py | 3 +++ optimum/exporters/openvino/model_configs.py | 6 ++++++ optimum/exporters/openvino/model_patcher.py | 23 +++++++++++++++++++++ 3 files changed, 32 insertions(+) diff --git a/optimum/exporters/openvino/convert.py b/optimum/exporters/openvino/convert.py index df2885fd0..4ed66b6c1 100644 --- a/optimum/exporters/openvino/convert.py +++ b/optimum/exporters/openvino/convert.py @@ -389,7 +389,10 @@ def ts_patched_forward(*args, **kwargs): if patch_16bit_model: from openvino.frontend.pytorch.patch_model import __make_16bit_traceable + # frontend may riases confusing warnings about modules already patched if model splitted on several parts + logging.disable(logging.WARNING) __make_16bit_traceable(model) + logging.disable(logging.NOTSET) check_dummy_inputs_are_allowed(model, dummy_inputs) input_info = _get_input_info(model, config, dummy_inputs) ov_model = convert_model( diff --git a/optimum/exporters/openvino/model_configs.py b/optimum/exporters/openvino/model_configs.py index 9dbcacb7f..1f009bc79 100644 --- a/optimum/exporters/openvino/model_configs.py +++ b/optimum/exporters/openvino/model_configs.py @@ -69,6 +69,7 @@ GptNeoxJapaneseModelPatcher, GptNeoxModelPatcher, IBertModelPatcher, + InputEmbeddingPatcher, InternLM2Patcher, InternLMModelPatcher, InternVLChatImageEmbeddingModelPatcher, @@ -1264,6 +1265,11 @@ def rename_ambiguous_inputs(self, inputs): model_inputs["input"] = inputs["input_ids"] return model_inputs + def patch_model_for_export( + self, model: Union["PreTrainedModel", "TFPreTrainedModel"], model_kwargs: Optional[Dict[str, Any]] = None + ) -> "ModelPatcher": + return InputEmbeddingPatcher(self, model, model_kwargs) + class LlavaConfigBehavior(str, enum.Enum): LANGUAGE = "language" diff --git a/optimum/exporters/openvino/model_patcher.py b/optimum/exporters/openvino/model_patcher.py index 8507d94fe..d8122de7d 100644 --- a/optimum/exporters/openvino/model_patcher.py +++ b/optimum/exporters/openvino/model_patcher.py @@ -2765,6 +2765,7 @@ def __exit__(self, exc_type, exc_value, traceback): self._model.pos_embed.forward = self._model.pos_embed._orig_forward +<<<<<<< HEAD def _minicpmv_resampler_forward(self, image_feature, pos_embed, key_padding_mask): bs = image_feature.shape[0] image_feature = self.kv_proj(image_feature) # B * L * D @@ -2991,3 +2992,25 @@ def __init__( def __exit__(self, exc_type, exc_value, traceback): super().__exit__(exc_type, exc_value, traceback) self._model.forward = self._model.__orig_forward + + +class InputEmbeddingPatcher(ModelPatcher): + def __init__( + self, + config: "OnnxConfig", + model: Union["PreTrainedModel", "TFPreTrainedModel"], + model_kwargs: Dict[str, Any], + ): + # making 16bit tracable overrides embeedings input signature these changes required to prevent this issue + model.__orig_forward = model.forward + + def forward(self, input): + return self.__orig_forward(input) + + model.forward = types.MethodType(forward, model) + + super().__init__(config, model, model_kwargs) + + def __exit__(self, exc_type, exc_value, traceback): + super().__exit__(exc_type, exc_value, traceback) + self._model.forward = self._model.__orig_forward From 00f22b92a885923e9cee4dce13fed910ee46037c Mon Sep 17 00:00:00 2001 From: eaidova Date: Tue, 5 Nov 2024 16:55:27 +0400 Subject: [PATCH 2/4] fix rebasing issue --- optimum/exporters/openvino/model_patcher.py | 1 - 1 file changed, 1 deletion(-) diff --git a/optimum/exporters/openvino/model_patcher.py b/optimum/exporters/openvino/model_patcher.py index d8122de7d..9f0e87671 100644 --- a/optimum/exporters/openvino/model_patcher.py +++ b/optimum/exporters/openvino/model_patcher.py @@ -2765,7 +2765,6 @@ def __exit__(self, exc_type, exc_value, traceback): self._model.pos_embed.forward = self._model.pos_embed._orig_forward -<<<<<<< HEAD def _minicpmv_resampler_forward(self, image_feature, pos_embed, key_padding_mask): bs = image_feature.shape[0] image_feature = self.kv_proj(image_feature) # B * L * D From b8e58535ec0e486542f14d471d225455425a8573 Mon Sep 17 00:00:00 2001 From: eaidova Date: Thu, 7 Nov 2024 08:17:03 +0400 Subject: [PATCH 3/4] apply review comments --- optimum/exporters/openvino/convert.py | 3 --- optimum/exporters/openvino/model_configs.py | 1 + optimum/exporters/openvino/model_patcher.py | 1 - tests/openvino/utils_tests.py | 2 +- 4 files changed, 2 insertions(+), 5 deletions(-) diff --git a/optimum/exporters/openvino/convert.py b/optimum/exporters/openvino/convert.py index 4ed66b6c1..df2885fd0 100644 --- a/optimum/exporters/openvino/convert.py +++ b/optimum/exporters/openvino/convert.py @@ -389,10 +389,7 @@ def ts_patched_forward(*args, **kwargs): if patch_16bit_model: from openvino.frontend.pytorch.patch_model import __make_16bit_traceable - # frontend may riases confusing warnings about modules already patched if model splitted on several parts - logging.disable(logging.WARNING) __make_16bit_traceable(model) - logging.disable(logging.NOTSET) check_dummy_inputs_are_allowed(model, dummy_inputs) input_info = _get_input_info(model, config, dummy_inputs) ov_model = convert_model( diff --git a/optimum/exporters/openvino/model_configs.py b/optimum/exporters/openvino/model_configs.py index 1f009bc79..5276ade33 100644 --- a/optimum/exporters/openvino/model_configs.py +++ b/optimum/exporters/openvino/model_configs.py @@ -1268,6 +1268,7 @@ def rename_ambiguous_inputs(self, inputs): def patch_model_for_export( self, model: Union["PreTrainedModel", "TFPreTrainedModel"], model_kwargs: Optional[Dict[str, Any]] = None ) -> "ModelPatcher": + # making 16bit tracable overrides embeedings input signature these changes required to prevent this issue return InputEmbeddingPatcher(self, model, model_kwargs) diff --git a/optimum/exporters/openvino/model_patcher.py b/optimum/exporters/openvino/model_patcher.py index 9f0e87671..dbbfb5662 100644 --- a/optimum/exporters/openvino/model_patcher.py +++ b/optimum/exporters/openvino/model_patcher.py @@ -3000,7 +3000,6 @@ def __init__( model: Union["PreTrainedModel", "TFPreTrainedModel"], model_kwargs: Dict[str, Any], ): - # making 16bit tracable overrides embeedings input signature these changes required to prevent this issue model.__orig_forward = model.forward def forward(self, input): diff --git a/tests/openvino/utils_tests.py b/tests/openvino/utils_tests.py index 2d2d71330..501d7ed94 100644 --- a/tests/openvino/utils_tests.py +++ b/tests/openvino/utils_tests.py @@ -84,7 +84,7 @@ "marian": "sshleifer/tiny-marian-en-de", "mbart": "hf-internal-testing/tiny-random-mbart", "minicpm": "katuni4ka/tiny-random-minicpm", - "minicpmv": "katuni4ka/tiny-random-minicpmv-2_6", + "minicpmv": "/home/ea/work/my_optimum_intel/optimum-intel/tiny-random-minicpmv-2_6", "mistral": "echarlaix/tiny-random-mistral", "mistral-nemo": "katuni4ka/tiny-random-mistral-nemo", "mixtral": "TitanML/tiny-mixtral", From 165817b564a9966cde3aa3657e85c2237946d4a7 Mon Sep 17 00:00:00 2001 From: Ekaterina Aidova Date: Fri, 8 Nov 2024 10:18:36 +0400 Subject: [PATCH 4/4] Update tests/openvino/utils_tests.py --- tests/openvino/utils_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/openvino/utils_tests.py b/tests/openvino/utils_tests.py index 501d7ed94..2d2d71330 100644 --- a/tests/openvino/utils_tests.py +++ b/tests/openvino/utils_tests.py @@ -84,7 +84,7 @@ "marian": "sshleifer/tiny-marian-en-de", "mbart": "hf-internal-testing/tiny-random-mbart", "minicpm": "katuni4ka/tiny-random-minicpm", - "minicpmv": "/home/ea/work/my_optimum_intel/optimum-intel/tiny-random-minicpmv-2_6", + "minicpmv": "katuni4ka/tiny-random-minicpmv-2_6", "mistral": "echarlaix/tiny-random-mistral", "mistral-nemo": "katuni4ka/tiny-random-mistral-nemo", "mixtral": "TitanML/tiny-mixtral",