[Bounding Boxes] Add Support for Bounding Box Transformations during Image Resizing (#20368)

* bounding box utils base

* add missing args and correct api call

* add support for diff formats

* function name correction

* use common methods

* add missing args for existing bounding box transformation methods

* use `get_random_transformation` for input height and width.
Remove input dict args for default orig_height and orig_width

* Correct single bounding box transform input and output format
Change layer names
Bounding box resizing as per the aspect ratio

* handle when pad_to_aspect_ratio is False

* use clip for bounding box transformation and correct the dtype while mul op

* corrrect key for labels

* correct the stacking of boxes

* nit

* correct box transformations

* handle without aspect ratio

* only pad to aspect ratio is handled as of now

* handle crop to aspect ratio flag for bboxes transformation

* Add test cases where and simplify the bbox transform function

* fix test cases for torch channels first

Author: sineeli

keras/api/_tf_keras/keras/layers/__init__.py

@@ -145,6 +145,9 @@
 from keras.src.layers.preprocessing.image_preprocessing.center_crop import (
     CenterCrop,
 )
+from keras.src.layers.preprocessing.image_preprocessing.max_num_bounding_box import (
+    MaxNumBoundingBoxes,
+)
 from keras.src.layers.preprocessing.image_preprocessing.random_brightness import (
     RandomBrightness,
 )

keras/api/_tf_keras/keras/utils/__init__.py

@@ -4,6 +4,7 @@
 since your modifications would be overwritten.
 """
 
+from keras.api.utils import bounding_boxes
 from keras.api.utils import legacy
 from keras.src.backend.common.global_state import clear_session
 from keras.src.backend.common.keras_tensor import is_keras_tensor

keras/api/_tf_keras/keras/utils/bounding_boxes/__init__.py

@@ -0,0 +1,21 @@
+"""DO NOT EDIT.
+
+This file was autogenerated. Do not edit it by hand,
+since your modifications would be overwritten.
+"""
+
+from keras.src.layers.preprocessing.image_preprocessing.bounding_boxes.converters import (
+    affine_transform,
+)
+from keras.src.layers.preprocessing.image_preprocessing.bounding_boxes.converters import (
+    clip_to_image_size,
+)
+from keras.src.layers.preprocessing.image_preprocessing.bounding_boxes.converters import (
+    convert_format,
+)
+from keras.src.layers.preprocessing.image_preprocessing.bounding_boxes.converters import (
+    crop,
+)
+from keras.src.layers.preprocessing.image_preprocessing.bounding_boxes.converters import (
+    pad,
+)

keras/api/layers/__init__.py

@@ -145,6 +145,9 @@
 from keras.src.layers.preprocessing.image_preprocessing.center_crop import (
     CenterCrop,
 )
+from keras.src.layers.preprocessing.image_preprocessing.max_num_bounding_box import (
+    MaxNumBoundingBoxes,
+)
 from keras.src.layers.preprocessing.image_preprocessing.random_brightness import (
     RandomBrightness,
 )

keras/api/utils/__init__.py

@@ -4,6 +4,7 @@
 since your modifications would be overwritten.
 """
 
+from keras.api.utils import bounding_boxes
 from keras.api.utils import legacy
 from keras.src.backend.common.global_state import clear_session
 from keras.src.backend.common.keras_tensor import is_keras_tensor

keras/api/utils/bounding_boxes/__init__.py

@@ -0,0 +1,21 @@
+"""DO NOT EDIT.
+
+This file was autogenerated. Do not edit it by hand,
+since your modifications would be overwritten.
+"""
+
+from keras.src.layers.preprocessing.image_preprocessing.bounding_boxes.converters import (
+    affine_transform,
+)
+from keras.src.layers.preprocessing.image_preprocessing.bounding_boxes.converters import (
+    clip_to_image_size,
+)
+from keras.src.layers.preprocessing.image_preprocessing.bounding_boxes.converters import (
+    convert_format,
+)
+from keras.src.layers.preprocessing.image_preprocessing.bounding_boxes.converters import (
+    crop,
+)
+from keras.src.layers.preprocessing.image_preprocessing.bounding_boxes.converters import (
+    pad,
+)

keras/src/layers/__init__.py

@@ -88,6 +88,9 @@
 from keras.src.layers.preprocessing.image_preprocessing.center_crop import (
     CenterCrop,
 )
+from keras.src.layers.preprocessing.image_preprocessing.max_num_bounding_box import (
+    MaxNumBoundingBoxes,
+)
 from keras.src.layers.preprocessing.image_preprocessing.random_brightness import (
     RandomBrightness,
 )

keras/src/layers/preprocessing/image_preprocessing/auto_contrast.py

@@ -88,7 +88,10 @@ def transform_labels(self, labels, transformation, training=True):
         return labels
 
     def transform_bounding_boxes(
-        self, bounding_boxes, transformation, training=True
+        self,
+        bounding_boxes,
+        transformation,
+        training=True,
     ):
         return bounding_boxes
 

keras/src/layers/preprocessing/image_preprocessing/base_image_preprocessing_layer.py

@@ -64,7 +64,10 @@ def transform_labels(self, labels, transformation, training=True):
         raise NotImplementedError()
 
     def transform_bounding_boxes(
-        self, bounding_boxes, transformation, training=True
+        self,
+        bounding_boxes,
+        transformation,
+        training=True,
     ):
         raise NotImplementedError()
 
@@ -88,13 +91,19 @@ def transform_single_label(self, label, transformation, training=True):
         return self.backend.numpy.squeeze(outputs, axis=0)
 
     def transform_single_bounding_box(
-        self, bounding_box, transformation, training=True
+        self,
+        bounding_box,
+        transformation,
+        training=True,
     ):
-        bounding_boxes = self.backend.numpy.expand_dims(bounding_box, axis=0)
+        bounding_boxes = self._format_single_input_bounding_box(bounding_box)
         outputs = self.transform_bounding_boxes(
-            bounding_boxes, transformation=transformation, training=training
+            bounding_boxes,
+            transformation=transformation,
+            training=training,
         )
-        return self.backend.numpy.squeeze(outputs, axis=0)
+        bounding_box = self._format_single_output_bounding_box(outputs)
+        return bounding_box
 
     def transform_single_segmentation_mask(
         self, segmentation_mask, transformation, training=True
@@ -144,8 +153,11 @@ def call(self, data, training=True):
                         "`bounding_box_format='xyxy'`."
                     )
                 bounding_boxes = densify_bounding_boxes(
-                    data["bounding_boxes"], backend=self.backend
+                    data["bounding_boxes"],
+                    is_batched=is_batched,
+                    backend=self.backend,
                 )
+
                 if is_batched:
                     data["bounding_boxes"] = self.transform_bounding_boxes(
                         bounding_boxes,
@@ -203,6 +215,32 @@ def call(self, data, training=True):
             training=training,
         )
 
+    def _format_single_input_bounding_box(self, bounding_box):
+        for key in bounding_box:
+            if key == "labels":
+                bounding_box[key] = self.backend.numpy.expand_dims(
+                    bounding_box[key], axis=0
+                )
+            if key == "boxes":
+                bounding_box[key] = self.backend.numpy.expand_dims(
+                    bounding_box[key], axis=0
+                )
+
+        return bounding_box
+
+    def _format_single_output_bounding_box(self, bounding_boxes):
+        for key in bounding_boxes:
+            if key == "labels":
+                bounding_boxes[key] = self.backend.numpy.squeeze(
+                    bounding_boxes[key], axis=0
+                )
+            if key == "boxes":
+                bounding_boxes[key] = self.backend.numpy.squeeze(
+                    bounding_boxes[key], axis=0
+                )
+
+        return bounding_boxes
+
     def get_config(self):
         config = super().get_config()
         if self.bounding_box_format is not None: