From cf4aece8bbe5926faea3c727fff9d9e360a33988 Mon Sep 17 00:00:00 2001 From: dlyakhov Date: Thu, 8 Aug 2024 14:54:03 +0200 Subject: [PATCH] Swin transformer conformance test --- tests/post_training/data/ptq_reference_data.yaml | 6 ++++++ tests/post_training/model_scope.py | 11 +++++++++++ .../pipelines/image_classification_torchvision.py | 7 ++++--- tests/post_training/test_quantize_conformance.py | 9 +++++---- 4 files changed, 26 insertions(+), 7 deletions(-) diff --git a/tests/post_training/data/ptq_reference_data.yaml b/tests/post_training/data/ptq_reference_data.yaml index 490cb7e73da..17321b06036 100644 --- a/tests/post_training/data/ptq_reference_data.yaml +++ b/tests/post_training/data/ptq_reference_data.yaml @@ -34,6 +34,12 @@ torchvision/resnet18_backend_CUDA_TORCH: metric_value: 0.69152 torchvision/resnet18_backend_FX_TORCH: metric_value: 0.6946 +torchvision/swin_v2_s_backend_FP32: + metric_value: 0.83712 +torchvision/swin_v2_s_backend_OV: + metric_value: 0.83638 +torchvision/swin_v2_s_backend_FX_TORCH: + metric_value: 0.82908 timm/crossvit_9_240_backend_CUDA_TORCH: metric_value: 0.689 timm/crossvit_9_240_backend_FP32: diff --git a/tests/post_training/model_scope.py b/tests/post_training/model_scope.py index 7f78d70528c..1bec8d1d4ae 100644 --- a/tests/post_training/model_scope.py +++ b/tests/post_training/model_scope.py @@ -75,6 +75,17 @@ "backends": [BackendType.FX_TORCH, BackendType.TORCH, BackendType.CUDA_TORCH, BackendType.OV, BackendType.ONNX], "batch_size": 128, }, + { + "reported_name": "torchvision/swin_v2_s", + "model_id": "swin_v2_s", + "pipeline_cls": ImageClassificationTorchvision, + "compression_params": { + "model_type": ModelType.TRANSFORMER, + "advanced_parameters": AdvancedQuantizationParameters(smooth_quant_alpha=0.5), + }, + "backends": [BackendType.FX_TORCH, BackendType.OV], + "batch_size": 1, + }, # Timm models { "reported_name": "timm/crossvit_9_240", diff --git a/tests/post_training/pipelines/image_classification_torchvision.py b/tests/post_training/pipelines/image_classification_torchvision.py index 91e586605cb..c42aa9ab1bb 100644 --- a/tests/post_training/pipelines/image_classification_torchvision.py +++ b/tests/post_training/pipelines/image_classification_torchvision.py @@ -43,13 +43,14 @@ def prepare_model(self) -> None: model = model_cls(weights=self.model_weights) model.eval() - self.static_input_size = [self.batch_size, 3, 224, 224] + default_input_size = [self.batch_size, 3, 224, 224] + self.dummy_tensor = self.model_weights.transforms()(torch.rand(default_input_size)) + self.static_input_size = list(self.dummy_tensor.shape) + self.input_size = self.static_input_size.copy() if self.batch_size > 1: # Dynamic batch_size shape export self.input_size[0] = -1 - self.dummy_tensor = torch.rand(self.static_input_size) - if self.backend == BackendType.FX_TORCH: with torch.no_grad(): with disable_patching(): diff --git a/tests/post_training/test_quantize_conformance.py b/tests/post_training/test_quantize_conformance.py index 368664de3da..d218aeb78a4 100644 --- a/tests/post_training/test_quantize_conformance.py +++ b/tests/post_training/test_quantize_conformance.py @@ -247,7 +247,8 @@ def test_ptq_quantization( err_msg = None test_model_param = None start_time = time.perf_counter() - try: + if True: + # try: if test_case_name not in ptq_reference_data: raise nncf.ValidationError(f"{test_case_name} does not exist in 'reference_data.yaml'") test_model_param = PTQ_TEST_CASES[test_case_name] @@ -271,9 +272,9 @@ def test_ptq_quantization( ) pipeline: BaseTestPipeline = pipeline_cls(**pipeline_kwargs) pipeline.run() - except Exception as e: - err_msg = str(e) - traceback.print_exc() + # except Exception as e: + # err_msg = str(e) + # traceback.print_exc() if pipeline is not None: pipeline.cleanup_cache()