Fix naming of statistics in MeanAveragePrecision with custom max det thresholds (Lightning-AI#2367)

* fix src code + fix docs
* fix tests

Author: SkafteNicki

CHANGELOG.md

@@ -36,6 +36,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Fixed cached network in `FeatureShare` not being moved to the correct device ([#2348](https://github.com/Lightning-AI/torchmetrics/pull/2348))
 
 
+- Fix naming of statistics in `MeanAveragePrecision` with custom max det thresholds ([#2367](https://github.com/Lightning-AI/torchmetrics/pull/2367))
+
+
 - Fixed custom aggregation in retrieval metrics ([#2364](https://github.com/Lightning-AI/torchmetrics/pull/2364))
 
 

src/torchmetrics/detection/mean_ap.py

@@ -128,9 +128,12 @@ class MeanAveragePrecision(Metric):
         - map_small: (:class:`~torch.Tensor`), mean average precision for small objects
         - map_medium:(:class:`~torch.Tensor`), mean average precision for medium objects
         - map_large: (:class:`~torch.Tensor`), mean average precision for large objects
-        - mar_1: (:class:`~torch.Tensor`), mean average recall for 1 detection per image
-        - mar_10: (:class:`~torch.Tensor`), mean average recall for 10 detections per image
-        - mar_100: (:class:`~torch.Tensor`), mean average recall for 100 detections per image
+        - mar_{mdt[0]}: (:class:`~torch.Tensor`), mean average recall for `max_detection_thresholds[0]` (default 1)
+          detection per image
+        - mar_{mdt[1]}: (:class:`~torch.Tensor`), mean average recall for `max_detection_thresholds[1]` (default 10)
+          detection per image
+        - mar_{mdt[1]}: (:class:`~torch.Tensor`), mean average recall for `max_detection_thresholds[2]` (default 100)
+          detection per image
         - mar_small: (:class:`~torch.Tensor`), mean average recall for small objects
         - mar_medium: (:class:`~torch.Tensor`), mean average recall for medium objects
         - mar_large: (:class:`~torch.Tensor`), mean average recall for large objects
@@ -140,8 +143,8 @@ class MeanAveragePrecision(Metric):
           IoU=0.75
         - map_per_class: (:class:`~torch.Tensor`) (-1 if class metrics are disabled), mean average precision per
           observed class
-        - mar_100_per_class: (:class:`~torch.Tensor`) (-1 if class metrics are disabled), mean average recall for 100
-          detections per image per observed class
+        - mar_{mdt[2]}_per_class: (:class:`~torch.Tensor`) (-1 if class metrics are disabled), mean average recall for
+          `max_detection_thresholds[2]` (default 100) detections per image per observed class
         - classes (:class:`~torch.Tensor`), list of all observed classes
 
     For an example on how to use this metric check the `torchmetrics mAP example`_.
@@ -184,8 +187,7 @@ class MeanAveragePrecision(Metric):
             with step ``0.01``. Else provide a list of floats.
         max_detection_thresholds:
             Thresholds on max detections per image. If set to `None` will use thresholds ``[1, 10, 100]``.
-            Else, please provide a list of ints. If the `pycocotools` backend is used then the list needs to have
-            length 3. If this is a problem, shift to `faster_coco_eval` which supports more detection thresholds.
+            Else, please provide a list of ints of length 3, which is the only supported length by both backends.
         class_metrics:
             Option to enable per-class metrics for mAP and mAR_100. Has a performance impact that scales linearly with
             the number of classes in the dataset.
@@ -410,10 +412,10 @@ def __init__(
                 f"Expected argument `max_detection_thresholds` to either be `None` or a list of ints"
                 f" but got {max_detection_thresholds}"
             )
-        if max_detection_thresholds is not None and backend == "pycocotools" and len(max_detection_thresholds) != 3:
+        if max_detection_thresholds is not None and len(max_detection_thresholds) != 3:
             raise ValueError(
-                "When using `pycocotools` backend the number of max detection thresholds should be 3 else"
-                f" it will not work correctly with the backend. Got value {len(max_detection_thresholds)}."
+                "When providing a list of max detection thresholds it should have length 3."
+                " Got value {len(max_detection_thresholds)}"
             )
         max_det_thr, _ = torch.sort(torch.tensor(max_detection_thresholds or [1, 10, 100], dtype=torch.int))
         self.max_detection_thresholds = max_det_thr.tolist()
@@ -556,7 +558,7 @@ def compute(self) -> dict:
                         coco_eval.params.maxDets = self.max_detection_thresholds
 
                     map_per_class_list = []
-                    mar_100_per_class_list = []
+                    mar_per_class_list = []
                     for class_id in self._get_classes():
                         coco_eval.params.catIds = [class_id]
                         with contextlib.redirect_stdout(io.StringIO()):
@@ -566,18 +568,18 @@ def compute(self) -> dict:
                             class_stats = coco_eval.stats
 
                         map_per_class_list.append(torch.tensor([class_stats[0]]))
-                        mar_100_per_class_list.append(torch.tensor([class_stats[8]]))
+                        mar_per_class_list.append(torch.tensor([class_stats[8]]))
 
                     map_per_class_values = torch.tensor(map_per_class_list, dtype=torch.float32)
-                    mar_100_per_class_values = torch.tensor(mar_100_per_class_list, dtype=torch.float32)
+                    mar_per_class_values = torch.tensor(mar_per_class_list, dtype=torch.float32)
                 else:
                     map_per_class_values = torch.tensor([-1], dtype=torch.float32)
-                    mar_100_per_class_values = torch.tensor([-1], dtype=torch.float32)
+                    mar_per_class_values = torch.tensor([-1], dtype=torch.float32)
                 prefix = "" if len(self.iou_type) == 1 else f"{i_type}_"
                 result_dict.update(
                     {
                         f"{prefix}map_per_class": map_per_class_values,
-                        f"{prefix}mar_100_per_class": mar_100_per_class_values,
+                        f"{prefix}mar_{self.max_detection_thresholds[-1]}_per_class": mar_per_class_values,
                     },
                 )
         result_dict.update({"classes": torch.tensor(self._get_classes(), dtype=torch.int32)})
@@ -616,19 +618,19 @@ def _get_coco_datasets(self, average: Literal["macro", "micro"]) -> Tuple[object
 
         return coco_preds, coco_target
 
-    @staticmethod
-    def _coco_stats_to_tensor_dict(stats: List[float], prefix: str) -> Dict[str, Tensor]:
+    def _coco_stats_to_tensor_dict(self, stats: List[float], prefix: str) -> Dict[str, Tensor]:
         """Converts the output of COCOeval.stats to a dict of tensors."""
+        mdt = self.max_detection_thresholds
         return {
             f"{prefix}map": torch.tensor([stats[0]], dtype=torch.float32),
             f"{prefix}map_50": torch.tensor([stats[1]], dtype=torch.float32),
             f"{prefix}map_75": torch.tensor([stats[2]], dtype=torch.float32),
             f"{prefix}map_small": torch.tensor([stats[3]], dtype=torch.float32),
             f"{prefix}map_medium": torch.tensor([stats[4]], dtype=torch.float32),
             f"{prefix}map_large": torch.tensor([stats[5]], dtype=torch.float32),
-            f"{prefix}mar_1": torch.tensor([stats[6]], dtype=torch.float32),
-            f"{prefix}mar_10": torch.tensor([stats[7]], dtype=torch.float32),
-            f"{prefix}mar_100": torch.tensor([stats[8]], dtype=torch.float32),
+            f"{prefix}mar_{mdt[0]}": torch.tensor([stats[6]], dtype=torch.float32),
+            f"{prefix}mar_{mdt[1]}": torch.tensor([stats[7]], dtype=torch.float32),
+            f"{prefix}mar_{mdt[2]}": torch.tensor([stats[8]], dtype=torch.float32),
             f"{prefix}mar_small": torch.tensor([stats[9]], dtype=torch.float32),
             f"{prefix}mar_medium": torch.tensor([stats[10]], dtype=torch.float32),
             f"{prefix}mar_large": torch.tensor([stats[11]], dtype=torch.float32),

tests/unittests/detection/test_map.py

@@ -861,6 +861,10 @@ def test_many_detection_thresholds(self, backend):
         else:
             assert round(res["map"].item(), 5) == 0.6
 
+        assert "mar_1" in res
+        assert "mar_10" in res
+        assert "mar_1000" in res
+
     @pytest.mark.parametrize("max_detection_thresholds", [[1, 10], [1, 10, 50, 100]])
     def test_with_more_and_less_detection_thresholds(self, max_detection_thresholds, backend):
         """Test how metric is working when list of max detection thresholds is not 3.
@@ -869,25 +873,7 @@ def test_with_more_and_less_detection_thresholds(self, max_detection_thresholds,
         https://github.com/ppwwyyxx/cocoapi/blob/master/PythonAPI/pycocotools/cocoeval.py#L461
 
         """
-        preds = [
-            {
-                "boxes": torch.tensor([[258.0, 41.0, 606.0, 285.0]]),
-                "scores": torch.tensor([0.536]),
-                "labels": torch.tensor([0]),
-            }
-        ]
-        target = [
-            {
-                "boxes": torch.tensor([[214.0, 41.0, 562.0, 285.0]]),
-                "labels": torch.tensor([0]),
-            }
-        ]
-
-        if backend == "pycocotools":
-            with pytest.raises(
-                ValueError, match="When using `pycocotools` backend the number of max detection thresholds should.*"
-            ):
-                metric = MeanAveragePrecision(max_detection_thresholds=max_detection_thresholds, backend=backend)
-        else:
-            metric = MeanAveragePrecision(max_detection_thresholds=max_detection_thresholds, backend=backend)
-            metric(preds, target)
+        with pytest.raises(
+            ValueError, match="When providing a list of max detection thresholds it should have length 3.*"
+        ):
+            MeanAveragePrecision(max_detection_thresholds=max_detection_thresholds, backend=backend)