[GSK-2569, GSK-2585, GSK-2658] Scan for computer vision

examples/landmark_detection/example_scan.py

@@ -0,0 +1,19 @@
+# %%
+from giskard_vision.landmark_detection.models.wrappers import OpenCVWrapper
+from giskard_vision.landmark_detection.dataloaders.loaders import DataLoader300W
+
+from giskard_vision.scanner import scan
+
+# %%
+model = OpenCVWrapper()
+dl_ref = DataLoader300W(dir_path="./datasets/300W/sample/")
+
+results = scan(model, dl_ref)
+
+
+# %%
+
+results.to_html(filename="example_vision_300w.html")
+# %%
+
+# %%

giskard_vision/__init__.py

@@ -1,5 +1,5 @@
+from giskard_vision.landmark_detection import detectors
+
 from . import landmark_detection
 
-__all__ = [
-    "landmark_detection",
-]
+__all__ = ["landmark_detection", "detectors"]

giskard_vision/detectors/base.py

@@ -0,0 +1,162 @@
+from abc import abstractmethod
+from dataclasses import dataclass
+from typing import Any, List, Optional, Sequence, Tuple
+
+from ..utils.errors import GiskardImportError
+
+
+@dataclass
+class ScanResult:
+    """
+    Minimum requirement for Issues to work properly
+
+    Attributes:
+        name (str): Details on transformation or slice
+        group (str): Group name for the issue
+        metric_name (str): Name of the metric
+        metric_value (float): Value of the metric on sliced or transformed dataset
+        metric_reference_value (float): Value of the metric on original dataset
+        issue_level (IssueLevel): Level of issue [Issue.MAJOR, Issue.MEDIUM, Issue.MINOR]
+        slice_size (int): Number of samples in sliced or transformed dataset
+
+    Methods:
+        get_meta_required() -> dict:
+            Returns a dictionary containing meta information required by giskard Issue
+    """
+
+    name: str
+    group: str
+    metric_name: str
+    metric_value: float
+    metric_reference_value: float
+    issue_level: str
+    slice_size: int
+    filename_examples: Optional[Sequence[str]]
+    relative_delta: float
+
+    def get_meta_required(self) -> dict:
+        # Get the meta required by the original scan API
+        deviation = f"{self.relative_delta * 100:+.2f}% than global"
+        return {
+            "metric": self.metric_name,
+            "metric_value": self.metric_value,
+            "metric_reference_value": self.metric_reference_value,
+            "deviation": deviation,
+            "slice_size": self.slice_size,
+        }
+
+
+class DetectorVisionBase:
+    """
+    Abstract class for Vision Detectors
+
+    Methods:
+        run(model: Any, dataset: Any, features: Optional[Any], issue_levels: Tuple[IssueLevel]) -> Sequence[Issues]:
+            Returns a list of giskard Issue to feed to the scan.
+
+        get_issues(model: Any, dataset: Any, results: List[ScanResult], issue_levels: Tuple[IssueLevel]):
+            Returns a list of giskard Issue from results output by get_results
+
+        get_results(model: Any, dataset: Any) -> List[ScanResult]
+            Abstract method that returns a list of ScanResult objects containing
+            evaluation results for the scan.
+    """
+
+    group: str
+    warning_messages: dict
+    issue_level_threshold: float = 0.2
+    deviation_threshold: float = 0.05
+
+    def run(
+        self,
+        model: Any,
+        dataset: Any,
+        features: Optional[Any] = None,
+        issue_levels: Tuple[Any] = None,
+        embed: bool = True,
+    ) -> Sequence[Any]:
+        results = self.get_results(model, dataset)
+        issues = self.get_issues(model, dataset, results=results, issue_levels=issue_levels, embed=embed)
+        return issues
+
+    def get_issues(
+        self, model: Any, dataset: Any, results: List[ScanResult], issue_levels: Tuple[Any], embed: bool = True
+    ) -> Sequence[Any]:
+        """
+        Returns a list of giskard Issue from results output by get_results
+
+        Args:
+            model (Any): model
+            dataset (Any): dataset
+            results (List[ScanResult]): results output by get_results
+            issue_levels (Tuple[IssueLevel]): issue levels that will be displayed (IssueLevel.MAJOR, IssueLevel.MEDIUM)
+
+        Returns:
+            Sequence[Issue]
+        """
+
+        issues = []
+
+        try:
+            from giskard.scanner.issues import Issue, IssueGroup, IssueLevel
+
+            from .example_manager import ImagesExampleManager
+
+        except (ImportError, ModuleNotFoundError) as e:
+            raise GiskardImportError(["giskard"]) from e
+
+        if issue_levels is None:
+            issue_levels = (IssueLevel.MAJOR, IssueLevel.MEDIUM)
+
+        for result in results:
+            if result.issue_level in issue_levels:
+                issues.append(
+                    Issue(
+                        model,
+                        dataset,
+                        level=result.issue_level,
+                        slicing_fn=result.name,
+                        group=IssueGroup(
+                            result.group,
+                            self.warning_messages[result.group] if result.group in self.warning_messages else "",
+                        ),
+                        meta=result.get_meta_required(),
+                        scan_examples=ImagesExampleManager(result.filename_examples, embed=embed),
+                        display_footer_info=False,
+                    )
+                )
+
+        return issues
+
+    @abstractmethod
+    def get_results(self, model: Any, dataset: Any) -> List[ScanResult]:
+        """Returns a list of ScanResult
+        ScanResult should contain
+        name : str
+            Details on transformation or slice
+        group : IssueGroup
+            Group name for the issue
+        metric_name : str
+            Name of the metric
+        metric_value : float
+            Value of the metric on sliced or transformed dataset
+        metric_reference_value : float
+            Value of the metric on original dataset
+        issue_level : IssueLevel
+            Level of issue [Issue.MAJOR, Issue.MEDIUM, Issue.MINOR]
+        slice_size : int
+            Number of samples in sliced or transformed dataset
+
+        Parameters
+        ----------
+        model : Any
+            Model to be evaluated
+        dataset : Any
+            Dataset on which to evaluate the model
+
+        Returns
+        -------
+        List[ScanResult]
+            List of ScanResult objects containing evaluation results
+        """
+        ...

giskard_vision/detectors/example_manager.py

@@ -0,0 +1,76 @@
+import base64
+from abc import ABC, abstractmethod
+from copy import deepcopy
+from typing import Any
+
+import cv2
+
+
+class ExampleManager(ABC):
+    """
+    Abstract class to manage examples from different data types.
+    """
+
+    @abstractmethod
+    def add_examples(self, example: Any):
+        ...
+
+    @abstractmethod
+    def head(self, n: int):
+        ...
+
+    @abstractmethod
+    def to_html(self):
+        ...
+
+
+class ImagesExampleManager:
+    """
+    Class to manage images examples
+    """
+
+    def __init__(self, examples: list = [], embed: bool = True):
+        self.examples = examples
+        self.embed = embed
+
+    def add_examples(self, example):
+        if isinstance(example, list):
+            self.examples += example
+        else:
+            self.examples.append(example)
+
+    def head(self, n):
+        """
+        Returns a new example manager keeping only n first examples
+
+        Args:
+            n (int): number of elements to keep
+
+        Returns:
+            ExampleManager: new example manager with n first examples
+        """
+        return type(self)(deepcopy(self.examples[:n]), embed=self.embed)
+
+    def __len__(self):
+        return len(self.examples)
+
+    def to_html(self, **kwargs):
+        html = '<div style="display:flex;justify-content:space-around">'
+        for elt in self.examples:
+            html += f'<img src="data:image/png;base64,{self._encode(elt) if self.embed else elt}" style="width:30%">'
+        html += "</div>"
+        return html
+
+    def _encode(self, path_img: str):
+        """
+        Encode images into base64 to embed them in the html file
+
+        Args:
+            path_img (str): Path towards the image
+
+        Returns:
+           str: Image encoded in base64
+        """
+        img = cv2.imread(path_img)
+        png_img = cv2.imencode(".png", img)
+        return base64.b64encode(png_img[1]).decode("utf-8")

giskard_vision/landmark_detection/dataloaders/base.py

@@ -503,6 +503,8 @@ class DataLoaderWrapper(DataIteratorBase):
         _wrapped_dataloader (DataIteratorBase): The wrapped data loader instance.
     """
 
+    dataloader_type: str = "standard"
+
     def __init__(self, dataloader: DataIteratorBase) -> None:
         """
         Initializes the DataLoaderWrapper with a given DataIteratorBase instance.

giskard_vision/landmark_detection/dataloaders/loaders.py

@@ -5,7 +5,7 @@
 import cv2
 import numpy as np
 
-from giskard_vision.landmark_detection.utils.errors import GiskardImportError
+from giskard_vision.utils.errors import GiskardImportError
 
 from .base import DataIteratorBase, DataLoaderBase
 

giskard_vision/landmark_detection/dataloaders/wrappers.py

@@ -11,7 +11,7 @@
     resize_image,
     resize_marks,
 )
-from giskard_vision.landmark_detection.utils.errors import GiskardImportError
+from giskard_vision.utils.errors import GiskardImportError
 
 from .base import DataIteratorBase, DataLoaderWrapper, SingleLandmarkData
 
@@ -362,6 +362,8 @@ class FilteredDataLoader(DataLoaderWrapper):
         FilteredDataLoader: Filtered data loader instance.
     """
 
+    dataloader_type = "filter"
+
     @property
     def name(self):
         """

giskard_vision/landmark_detection/detectors/__init__.py

@@ -0,0 +1,15 @@
+from .cropping_detector import CroppingDetectorLandmark
+from .ethnicity_bias_detector import EthnicityDetectorLandmark
+from .head_pose_detector import HeadPoseDetectorLandmark
+from .transformation_blurring_detector import TransformationBlurringDetectorLandmark
+from .transformation_color_detector import TransformationColorDetectorLandmark
+from .transformation_resize_detector import TransformationResizeDetectorLandmark
+
+__all__ = [
+    "CroppingDetectorLandmark",
+    "HeadPoseDetectorLandmark",
+    "TransformationBlurringDetectorLandmark",
+    "TransformationColorDetectorLandmark",
+    "TransformationResizeDetectorLandmark",
+    "EthnicityDetectorLandmark",
+]