fix: Improve anomaly threshold usage in inference task (#67)

* fix: Use training threshold to generate reports instead of 0 for anomaly test with no gt

* feat: Use image training threshold automatically if no labels is provided for testing

Allow defining training threshold type from config

* docs: Update anomaly test documentation

* build: Bump version 1.2.5 -> 1.2.6

* docs: Update changelog

Approved By: @rcmalli

CHANGELOG.md

@@ -2,6 +2,15 @@
 # Changelog
 All notable changes to this project will be documented in this file.
 
+### [1.2.6]
+
+#### Added
+
+- Add optional `training_threshold_type` for anomaly detection inference task.
+#### Changed
+
+- Compute results using the training image threshold instead of zero when running anomaly inference with no labelled data.
+
 ### [1.2.5]
 
 #### Fixed

docs/tutorials/examples/anomaly_detection.md

@@ -290,9 +290,11 @@ datamodule:
 task:
   model_path: ???
   use_training_threshold: false
+  training_threshold_type: image
 ```
 
 By default, the inference will recompute the threshold based on test data to maximize the F1-score, if you want to use the threshold from the training phase you can set the `use_training_threshold` parameter to true.
+The `training_threshold_type` can be used to specify which training threshold to use, it can be either `image` or `pixel`, if not specified the `image` threshold will be used.
 
 The model path is the path to an exported model, at the moment `torchscript` and `onnx` models are supported (exported automatically after a training experiment). Right now only the `CFLOW` model is not supported for inference as it's not compatible with botyh torchscript and onnx.
 

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "quadra"
-version = "1.2.5"
+version = "1.2.6"
 description = "Deep Learning experiment orchestration library"
 authors = [
   { name = "Alessandro Polidori", email = "[email protected]" },
@@ -118,7 +118,7 @@ repository = "https://github.com/orobix/quadra"
 
 # Adapted from https://realpython.com/pypi-publish-python-package/#version-your-package
 [tool.bumpver]
-current_version = "1.2.5"
+current_version = "1.2.6"
 version_pattern = "MAJOR.MINOR.PATCH"
 commit_message = "build: Bump version {old_version} -> {new_version}"
 commit = true

quadra/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "1.2.5"
+__version__ = "1.2.6"
 
 
 def get_version():

quadra/configs/experiment/base/anomaly/inference.yaml

@@ -18,3 +18,4 @@ datamodule:
 task:
   model_path: ???
   use_training_threshold: false
+  training_threshold_type: image

quadra/tasks/anomaly.py

@@ -2,7 +2,7 @@
 import json
 import os
 from collections import Counter
-from typing import Dict, Generic, List, Optional, TypeVar, Union, cast
+from typing import Dict, Generic, List, Literal, Optional, TypeVar, Union, cast
 
 import cv2
 import hydra
@@ -297,12 +297,26 @@ class AnomalibEvaluation(Evaluation[AnomalyDataModule]):
     """
 
     def __init__(
-        self, config: DictConfig, model_path: str, use_training_threshold: bool = False, device: Optional[str] = None
+        self,
+        config: DictConfig,
+        model_path: str,
+        use_training_threshold: bool = False,
+        device: Optional[str] = None,
+        training_threshold_type: Optional[Literal["image", "pixel"]] = None,
     ):
         super().__init__(config=config, model_path=model_path, device=device)
 
         self.use_training_threshold = use_training_threshold
 
+        if training_threshold_type is not None and training_threshold_type not in ["image", "pixel"]:
+            raise ValueError("Training threshold type must be either image or pixel")
+
+        if training_threshold_type is None and use_training_threshold:
+            log.warning("Using training threshold but no training threshold type is provided, defaulting to image")
+            training_threshold_type = "image"
+
+        self.training_threshold_type = training_threshold_type
+
     def prepare(self) -> None:
         """Prepare the evaluation."""
         super().prepare()
@@ -349,7 +363,7 @@ def test(self) -> None:
         if any(x != -1 for x in image_labels):
             if self.use_training_threshold:
                 _image_labels = torch.tensor(image_labels)
-                threshold = torch.tensor(float(self.model_data["image_threshold"]))
+                threshold = torch.tensor(float(self.model_data[f"{self.training_threshold_type}_threshold"]))
                 known_labels = torch.where(_image_labels != -1)[0]
 
                 _image_labels = _image_labels[known_labels]
@@ -362,9 +376,9 @@ def test(self) -> None:
                 optimal_f1_score = optimal_f1.compute()
                 threshold = optimal_f1.threshold
         else:
-            log.warning("No ground truth available during evaluation, F1 score and threshold set to 0")
+            log.warning("No ground truth available during evaluation, use training image threshold for reporting")
             optimal_f1_score = torch.tensor(0)
-            threshold = torch.tensor(0)
+            threshold = torch.tensor(float(self.model_data["image_threshold"]))
 
         log.info("Computed F1 score: %s", optimal_f1_score.item())
         self.metadata["anomaly_scores"] = anomaly_scores