[GSK-2569, GSK-2585, GSK-2658] Scan for computer vision

examples/landmark_detection/example_scan.py

@@ -1,19 +1,19 @@
 # %%
-from pathlib import Path
-
 from giskard_vision.landmark_detection.models.wrappers import OpenCVWrapper
 from giskard_vision.landmark_detection.dataloaders.loaders import DataLoader300W
 
-from giskard_vision.scanner.scanner import Scanner
+from giskard_vision.scanner import scan
 
 # %%
 model = OpenCVWrapper()
-dl_ref = DataLoader300W(dir_path=str(Path(__file__).parent / "300W/sample"))
+dl_ref = DataLoader300W(dir_path="./datasets/300W/sample/")
+
+results = scan(model, dl_ref)
 
-scanner = Scanner()
-results = scanner.analyze(model, dl_ref)
 
 # %%
 
-results.to_html(filename="example_vision.html")
+results.to_html(filename="example_vision_300w.html")
+# %%
+
 # %%

giskard_vision/detectors/base.py

@@ -63,8 +63,9 @@ class DetectorVisionBase:
     """
 
     group: str
-    warning_messages: dict = {}
-    threshold: Optional[float] = 0.1
+    warning_messages: dict
+    issue_level_threshold: float = 0.2
+    deviation_threshold: float = 0.05
 
     def run(
         self,
@@ -101,30 +102,30 @@ def get_issues(
 
             from .example_manager import ImagesExampleManager
 
-            if issue_levels is None:
-                issue_levels = (IssueLevel.MAJOR, IssueLevel.MEDIUM)
-
-            for result in results:
-                if result.issue_level in issue_levels:
-                    issues.append(
-                        Issue(
-                            model,
-                            dataset,
-                            level=result.issue_level,
-                            slicing_fn=result.name,
-                            group=IssueGroup(
-                                result.group,
-                                self.warning_messages[result.group] if result.group in self.warning_messages else "",
-                            ),
-                            meta=result.get_meta_required(),
-                            scan_examples=ImagesExampleManager(result.filename_examples, embed=embed),
-                            display_footer_info=False,
-                        )
-                    )
-
         except (ImportError, ModuleNotFoundError) as e:
             raise GiskardImportError(["giskard"]) from e
 
+        if issue_levels is None:
+            issue_levels = (IssueLevel.MAJOR, IssueLevel.MEDIUM)
+
+        for result in results:
+            if result.issue_level in issue_levels:
+                issues.append(
+                    Issue(
+                        model,
+                        dataset,
+                        level=result.issue_level,
+                        slicing_fn=result.name,
+                        group=IssueGroup(
+                            result.group,
+                            self.warning_messages[result.group] if result.group in self.warning_messages else "",
+                        ),
+                        meta=result.get_meta_required(),
+                        scan_examples=ImagesExampleManager(result.filename_examples, embed=embed),
+                        display_footer_info=False,
+                    )
+                )
+
         return issues
 
     @abstractmethod

giskard_vision/landmark_detection/detectors/base.py

@@ -7,9 +7,16 @@
 
 from giskard_vision.detectors.base import DetectorVisionBase, ScanResult
 from giskard_vision.landmark_detection.tests.base import TestDiff
-from giskard_vision.landmark_detection.tests.performance import NMEMean, NMEs
+from giskard_vision.landmark_detection.tests.performance import NMEMean
 from giskard_vision.utils.errors import GiskardImportError
 
+WARNING_MESSAGES: dict = {
+    "Cropping": "Cropping involves evaluating the landmark detection model on specific face areas.",
+    "Ethical": "The data are filtered by ethnicity to detect ethical biases in the landmark detection model.",
+    "Head Pose": "The data are filtered by head pose to detect biases in the landmark detection model.",
+    "Robustness": "Images from the dataset are blurred, recolored and resized to test the robustness of the model to transformations.",
+}
+
 
 class LandmarkDetectionBaseDetector(DetectorVisionBase):
     """
@@ -27,12 +34,7 @@ class LandmarkDetectionBaseDetector(DetectorVisionBase):
             Convert TestResult to ScanResult
     """
 
-    warning_messages: dict = {
-        "Cropping": "Cropping involves evaluating the landmark detection model on specific face areas.",
-        "Ethical": "The data are filtered by ethnicity to detect ethical biases in the landmark detection model.",
-        "Head Pose": "The data are filtered by head pose to detect biases in the landmark detection model.",
-        "Robustness": "Images from the dataset are blurred, recolored and resized to test the robustness of the model to transformations.",
-    }
+    warning_messages: dict = WARNING_MESSAGES
 
     @abstractmethod
     def get_dataloaders(self, dataset: Any) -> Sequence[Any]:
@@ -43,7 +45,7 @@ def get_results(self, model: Any, dataset: Any) -> Sequence[ScanResult]:
 
         results = []
         for dl in dataloaders:
-            test_result = TestDiff(metric=NMEMean, threshold=1, metric_for_examples=NMEs).run(
+            test_result = TestDiff(metric=NMEMean, threshold=1).run(
                 model=model,
                 dataloader=dl,
                 dataloader_ref=dataset,
@@ -81,9 +83,9 @@ def get_scan_result(self, test_result, filename_examples, name, size_data) -> Sc
 
         relative_delta = (test_result.metric_value_test - test_result.metric_value_ref) / test_result.metric_value_ref
 
-        if relative_delta > self.threshold:
+        if relative_delta > self.issue_level_threshold:
             issue_level = IssueLevel.MAJOR
-        elif relative_delta > 0:
+        elif relative_delta > self.deviation_threshold:
             issue_level = IssueLevel.MEDIUM
         else:
             issue_level = IssueLevel.MINOR

giskard_vision/landmark_detection/tests/base.py

@@ -169,6 +169,20 @@ def definition(prediction_result: PredictionResult, marks: np.ndarray, **kwargs)
         """
         ...
 
+    @staticmethod
+    @abstractmethod
+    def rank_data(prediction_result: PredictionResult, marks: np.ndarray, **kwargs) -> List[int]:
+        """Abstract method to define how the mtric ranks data samples from worse to best
+
+        Args:
+            prediction_result (PredictionResult): The prediction result to evaluate.
+            marks (np.ndarray): Ground truth facial landmarks.
+
+        Returns:
+            List[int]: Indexes of data samples from worse to best
+        """
+        ...
+
     @classmethod
     def validation(cls, prediction_result: PredictionResult, marks: np.ndarray, **kwargs) -> None:
         """Validate the input types for the metric calculation.
@@ -265,7 +279,6 @@ class TestDiff:
     metric: Metric
     threshold: float
     relative: bool = True
-    metric_for_examples: Optional[Metric] = None
 
     def run(
         self,
@@ -301,9 +314,8 @@ def run(
         ground_truth_ref = dataloader_ref.all_marks
         metric_value_ref = self.metric.get(prediction_result_ref, ground_truth_ref)
 
-        if self.metric_for_examples is not None:
-            metrics_examples = self.metric_for_examples.get(prediction_result, ground_truth)
-            indexes = sorted(range(len(metrics_examples)), key=metrics_examples.__getitem__)[::-1]
+        if hasattr(self.metric, "rank_data"):
+            indexes = self.metric.rank_data(prediction_result, ground_truth)
         else:
             indexes = None
 

giskard_vision/landmark_detection/tests/performance.py

@@ -74,6 +74,12 @@ class NMEMean(Metric):
     def definition(prediction_result: PredictionResult, marks: np.ndarray):
         return np.nanmean(NMEs.get(prediction_result, marks))
 
+    @staticmethod
+    def rank_data(prediction_result: PredictionResult, marks: np.ndarray):
+        predictions = NMEs.get(prediction_result, marks)
+        indexes = sorted(range(len(predictions)), key=predictions.__getitem__)[::-1]
+        return indexes
+
 
 @dataclass
 class NMEStd(Metric):

giskard_vision/scanner/__init__.py

@@ -0,0 +1,49 @@
+from typing import Any, List, Optional
+
+from .scanner import Scanner
+
+
+def scan(
+    model,
+    dataset,
+    detectors: Optional[List[Any]] = None,
+    params=None,
+    only=None,
+    verbose=True,
+    raise_exceptions=False,
+):
+    """Automatically detects model vulnerabilities.
+
+    See :class:`Scanner` for more details.
+
+    Parameters
+    ----------
+    model : BaseModel
+        A Giskard model object.
+    dataset : Dataset
+        A Giskard dataset object.
+    detectors : List[Any]
+        A list of detectors to use for the scan. If not specified, all detectors that correspond to the model type will be used.
+    params : dict
+        Advanced scanner configuration. See :class:`Scanner` for more details.
+    only : list
+        A tag list to limit the scan to a subset of detectors. For example,
+        ``giskard.scan(model, dataset, only=["performance"])`` will only run detectors for performance issues.
+    verbose : bool
+        Whether to print detailed info messages. Enabled by default.
+    raise_exceptions : bool
+        Whether to raise an exception if detection errors are encountered. By default, errors are logged and
+        handled gracefully, without interrupting the scan.
+
+    Returns
+    -------
+    ScanReport
+        A scan report object containing the results of the scan.
+    """
+    scanner = Scanner(params, only=only)
+    return scanner.analyze(
+        model, dataset=dataset, detectors=detectors, verbose=verbose, raise_exceptions=raise_exceptions
+    )
+
+
+__all__ = ["scan", "Scanner"]

giskard_vision/scanner/scanner.py

@@ -5,8 +5,13 @@
 from time import perf_counter
 from typing import Any, Optional, Sequence
 
-from giskard.scanner.registry import DetectorRegistry
-from giskard.scanner.report import ScanReport
+from ..utils.errors import GiskardImportError
+
+try:
+    from giskard.scanner.registry import DetectorRegistry
+    from giskard.scanner.report import ScanReport
+except (ImportError, ModuleNotFoundError) as e:
+    raise GiskardImportError(["giskard"]) from e
 
 
 def warning(content: str):