Scan for object detection

giskard_vision/core/detectors/metadata_scan_detector.py

@@ -85,26 +85,27 @@ def get_results(self, model: Any, dataset: Any) -> List[ScanResult]:
 
             # For each slice found, get appropriate scan results with the metric
             for issue in results.issues:
-                current_data_slice = giskard_dataset.slice(issue.slicing_fn)
-                indices = list(current_data_slice.df.sort_values(by="metric", ascending=False)["index"].values)
-                if not self.check_slice_already_selected(issue.slicing_fn.meta.display_name, current_issues):
-                    current_issues.append(issue.slicing_fn.meta.display_name)
-                    filenames = (
-                        [dataset.get_image_path(int(idx)) for idx in indices[: self.num_images]]
-                        if hasattr(dataset, "get_image_path")
-                        else []
-                    )
-                    list_scan_results.append(
-                        self.get_scan_result(
-                            metric_value=current_data_slice.df["metric"].mean(),
-                            metric_reference_value=giskard_dataset.df["metric"].mean(),
-                            metric_name=self.metric.name,
-                            filename_examples=filenames,
-                            name=issue.slicing_fn.meta.display_name,
-                            size_data=len(current_data_slice.df),
-                            issue_group=meta.issue_group(issue.features[0]),
+                if issue.slicing_fn is not None:
+                    current_data_slice = giskard_dataset.slice(issue.slicing_fn)
+                    indices = list(current_data_slice.df.sort_values(by="metric", ascending=False)["index"].values)
+                    if not self.check_slice_already_selected(issue.slicing_fn.meta.display_name, current_issues):
+                        current_issues.append(issue.slicing_fn.meta.display_name)
+                        filenames = (
+                            [dataset.get_image_path(int(idx)) for idx in indices[: self.num_images]]
+                            if hasattr(dataset, "get_image_path")
+                            else []
+                        )
+                        list_scan_results.append(
+                            self.get_scan_result(
+                                metric_value=current_data_slice.df["metric"].mean(),
+                                metric_reference_value=giskard_dataset.df["metric"].mean(),
+                                metric_name=self.metric.name,
+                                filename_examples=filenames,
+                                name=issue.slicing_fn.meta.display_name,
+                                size_data=len(current_data_slice.df),
+                                issue_group=meta.issue_group(issue.features[0]),
+                            )
                         )
-                    )
 
         return list_scan_results
 
@@ -131,6 +132,8 @@ def get_giskard_results_from_surrogate(self, surrogate, model, df_for_scan, list
         prediction_function = self.get_prediction_function(surrogate, model, df_for_scan)
 
         # Create Giskard dataset and model, and get scan results
+        if list_categories is None:
+            list_categories = []
         giskard_dataset = Dataset(
             df=df_for_scan, target=f"target_{surrogate.name}", cat_columns=list_categories + ["index"]
         )
@@ -140,7 +143,9 @@ def get_giskard_results_from_surrogate(self, surrogate, model, df_for_scan, list
             feature_names=list_metadata + ["index"],
             classification_labels=model.classification_labels if self.type_task == "classification" else None,
         )
-        results = scan(giskard_model, giskard_dataset, max_issues_per_detector=None, verbose=False)
+        results = scan(
+            giskard_model, giskard_dataset, max_issues_per_detector=None, verbose=False, raise_exceptions=True
+        )
 
         return giskard_dataset, results
 
@@ -217,8 +222,9 @@ def get_df_for_scan(self, model: Any, dataset: Any, list_metadata: Sequence[str]
         # we need the metadata, labels and image path on an individual basis,
         # and sometimes the model may fail on an image.
         # TODO: make this cleaner and more efficient with batch computations
+        from tqdm import tqdm
 
-        for i in range(len(dataset)):
+        for i in tqdm(range(len(dataset))):
             try:
                 metadata = dataset.get_meta(i)
 

giskard_vision/object_detection/dataloaders/loaders.py

@@ -115,7 +115,7 @@ def get_meta(self, idx: int) -> MetaData | None:
             MetaData | None: Metadata associated with the image.
         """
         meta_list = ["domain", "country", "location", "development_stage"]
-        data = {self.ds[idx][elt] for elt in meta_list}
+        data = {elt: self.ds[idx][elt] for elt in meta_list}
 
         return MetaData(data, categories=meta_list)
 

giskard_vision/object_detection/detectors/__init__.py

@@ -0,0 +1,5 @@
+from .metadata_detector import MetaDataScanDetectorObjectDetection
+
+__all__ = [
+    "MetaDataScanDetectorObjectDetection",
+]

giskard_vision/object_detection/detectors/metadata_detector.py

@@ -0,0 +1,46 @@
+from giskard_vision.core.detectors.metadata_scan_detector import MetaDataScanDetector
+from giskard_vision.object_detection.detectors.surrogate_functions import (
+    SurrogateArea,
+    SurrogateAspectRatio,
+    SurrogateCenterMassX,
+    SurrogateCenterMassY,
+    SurrogateDistanceFromCenter,
+    SurrogateMeanIntensity,
+    SurrogateNormalizedHeight,
+    SurrogateNormalizedPerimeter,
+    SurrogateNormalizedWidth,
+    SurrogateRelativeBottomRightX,
+    SurrogateRelativeBottomRightY,
+    SurrogateRelativeTopLeftX,
+    SurrogateRelativeTopLeftY,
+    SurrogateStdIntensity,
+)
+from giskard_vision.object_detection.tests.performance import IoU
+
+from ...core.detectors.decorator import maybe_detector
+
+
+@maybe_detector("metadata_object_detection", tags=["vision", "object_detection", "metadata"])
+class MetaDataScanDetectorObjectDetection(MetaDataScanDetector):
+    surrogates = [
+        SurrogateCenterMassX,
+        SurrogateCenterMassY,
+        SurrogateArea,
+        SurrogateAspectRatio,
+        SurrogateMeanIntensity,
+        SurrogateStdIntensity,
+        SurrogateNormalizedHeight,
+        SurrogateNormalizedWidth,
+        SurrogateDistanceFromCenter,
+        SurrogateRelativeBottomRightX,
+        SurrogateRelativeBottomRightY,
+        SurrogateRelativeTopLeftX,
+        SurrogateRelativeTopLeftY,
+        SurrogateNormalizedPerimeter,
+    ]
+    metric = IoU
+    type_task = "regression"
+    metric_type = "absolute"
+    metric_direction = "better_higher"
+    deviation_threshold = 0.10
+    issue_level_threshold = 0.05

giskard_vision/object_detection/detectors/surrogate_functions.py

@@ -0,0 +1,159 @@
+import numpy as np
+
+from giskard_vision.core.detectors.metadata_scan_detector import Surrogate
+
+
+@staticmethod
+def center_mass_x(result, image):
+    x_min, y_min, x_max, y_max = result[0]["boxes"]
+    center_x = (x_min + x_max) / 2
+    return center_x / image.shape[0]
+
+
+SurrogateCenterMassX = Surrogate("center_mass_x", center_mass_x)
+
+
+@staticmethod
+def center_mass_y(result, image):
+    x_min, y_min, x_max, y_max = result[0]["boxes"]
+    center_y = (y_min + y_max) / 2
+    return center_y / image.shape[1]
+
+
+SurrogateCenterMassY = Surrogate("center_mass_y", center_mass_y)
+
+
+@staticmethod
+def area(result, image):
+    x_min, y_min, x_max, y_max = result[0]["boxes"]
+    area = (x_max - x_min) * (y_max - y_min)
+    return area / (image.shape[0] * image.shape[1])
+
+
+SurrogateArea = Surrogate("area", area)
+
+
+@staticmethod
+def aspect_ratio(result, image):
+    x_min, y_min, x_max, y_max = result[0]["boxes"]
+    width = x_max - x_min
+    height = y_max - y_min
+    return width / height
+
+
+SurrogateAspectRatio = Surrogate("aspect_ratio", aspect_ratio)
+
+
+@staticmethod
+def normalized_width(result, image):
+    x_min, y_min, x_max, y_max = result[0]["boxes"]
+    width = x_max - x_min
+    normalized_width = width / image.shape[1]
+    return normalized_width
+
+
+SurrogateNormalizedWidth = Surrogate("normalized_width", normalized_width)
+
+
+@staticmethod
+def normalized_height(result, image):
+    x_min, y_min, x_max, y_max = result[0]["boxes"]
+    height = y_max - y_min
+    normalized_height = height / image.shape[0]
+    return normalized_height
+
+
+SurrogateNormalizedHeight = Surrogate("normalized_height", normalized_height)
+
+
+@staticmethod
+def normalized_perimeter(result, image):
+    x_min, y_min, x_max, y_max = result[0]["boxes"]
+    width = x_max - x_min
+    height = y_max - y_min
+    perimeter = 2 * (width + height)
+    normalized_perimeter = perimeter / (2 * (image.shape[0] + image.shape[1]))
+    return normalized_perimeter
+
+
+SurrogateNormalizedPerimeter = Surrogate("normalized_perimeter", normalized_perimeter)
+
+
+@staticmethod
+def relative_top_left_x(result, image):
+    x_min, y_min, x_max, y_max = result[0]["boxes"]
+    relative_x = x_min / float(image.shape[0])
+    return relative_x
+
+
+SurrogateRelativeTopLeftX = Surrogate("relative_top_left_x", relative_top_left_x)
+
+
+@staticmethod
+def relative_top_left_y(result, image):
+    x_min, y_min, x_max, y_max = result[0]["boxes"]
+    relative_y = y_min / float(image.shape[1])
+    return relative_y
+
+
+SurrogateRelativeTopLeftY = Surrogate("relative_top_left_y", relative_top_left_y)
+
+
+@staticmethod
+def relative_bottom_right_x(result, image):
+    x_min, y_min, x_max, y_max = result[0]["boxes"]
+    relative_x = x_max / float(image.shape[0])
+    return relative_x
+
+
+SurrogateRelativeBottomRightX = Surrogate("relative_bottom_right_x", relative_bottom_right_x)
+
+
+@staticmethod
+def relative_bottom_right_y(result, image):
+    x_min, y_min, x_max, y_max = result[0]["boxes"]
+    relative_y = y_max / float(image.shape[1])
+    return relative_y
+
+
+SurrogateRelativeBottomRightY = Surrogate("relative_bottom_right_y", relative_bottom_right_y)
+
+
+@staticmethod
+def distance_from_center(result, image):
+    x_min, y_min, x_max, y_max = result[0]["boxes"]
+    center_x = (x_min + x_max) / 2
+    center_y = (y_min + y_max) / 2
+    image_center_x = image.shape[1] / 2
+    image_center_y = image.shape[0] / 2
+    distance = np.sqrt((center_x - image_center_x) ** 2 + (center_y - image_center_y) ** 2)
+    return distance
+
+
+SurrogateDistanceFromCenter = Surrogate("distance_from_center", distance_from_center)
+
+
+@staticmethod
+def mean_intensity(result, image):
+    x_min, y_min, x_max, y_max = result[0]["boxes"]
+    y_min = max(0, y_min)
+    x_min = max(0, x_min)
+    roi = image[int(y_min) : int(y_max), int(x_min) : int(x_max)]
+    mean_intensity = roi.mean()
+    return mean_intensity
+
+
+SurrogateMeanIntensity = Surrogate("mean_intensity", mean_intensity)
+
+
+@staticmethod
+def std_intensity(result, image):
+    x_min, y_min, x_max, y_max = result[0]["boxes"]
+    y_min = max(0, y_min)
+    x_min = max(0, x_min)
+    roi = image[int(y_min) : int(y_max), int(x_min) : int(x_max)]
+    std_intensity = roi.std()
+    return std_intensity
+
+
+SurrogateStdIntensity = Surrogate("std_intensity", std_intensity)

giskard_vision/object_detection/models/wrappers.py

@@ -213,6 +213,7 @@ def preprocessing(self, image):
 
 class RacoonDetection(ModelBase):
     model_weights: str = "racoon_detection.h5"
+    model_type: str = "object_detection"
     image_size: int = 128
     alpha: float = 1.0
 

giskard_vision/object_detection/tests/base.py

@@ -0,0 +1,22 @@
+from dataclasses import dataclass
+
+from giskard_vision.core.tests.base import MetricBase
+
+from ..types import Types
+
+
+@dataclass
+class Metric(MetricBase):
+    @classmethod
+    def validation(cls, prediction_result: Types.prediction_result, ground_truth: Types.label, **kwargs) -> None:
+        """Validate the input types for the metric calculation.
+
+        Args:
+            prediction_result (Types.prediction_result): The prediction result to evaluate.
+            labels (Dict[str, Iterable[float]]): Ground truth for object detection.
+
+        Raises:
+            ValueError: If the input types are incorrect.
+
+        """
+        pass

giskard_vision/object_detection/tests/performance.py

@@ -0,0 +1,48 @@
+from dataclasses import dataclass
+
+from ..types import Types
+from .base import Metric
+
+
+@dataclass
+class IoU(Metric):
+    """Intersection over Union distance between a prediction and a ground truth"""
+
+    name = "IoU"
+    description = "Intersection over Union"
+
+    @staticmethod
+    def definition(prediction_result: Types.prediction_result, ground_truth: Types.label):
+
+        # if prediction_result.prediction.item().get("labels") != ground_truth.item().get("labels"):
+        #     return 0
+
+        gt_box = prediction_result.prediction.item().get("boxes")
+        pred_box = ground_truth.item().get("boxes")
+
+        x1_min, y1_min, x1_max, y1_max = gt_box
+        x2_min, y2_min, x2_max, y2_max = pred_box
+
+        # Calculate the coordinates of the intersection rectangle
+        x_inter_min = max(x1_min, x2_min)
+        y_inter_min = max(y1_min, y2_min)
+        x_inter_max = min(x1_max, x2_max)
+        y_inter_max = min(y1_max, y2_max)
+
+        # Compute the area of the intersection rectangle
+        if x_inter_max < x_inter_min or y_inter_max < y_inter_min:
+            inter_area = 0
+        else:
+            inter_area = (x_inter_max - x_inter_min) * (y_inter_max - y_inter_min)
+
+        # Compute the area of both the prediction and ground-truth rectangles
+        box1_area = (x1_max - x1_min) * (y1_max - y1_min)
+        box2_area = (x2_max - x2_min) * (y2_max - y2_min)
+
+        # Compute the union area
+        union_area = box1_area + box2_area - inter_area
+
+        # Compute the IoU
+        iou = inter_area / union_area
+
+        return iou