Bikerack filtering (#83)

* spellcheck

* add points_in_box() method to check whether points are inside the box

* test points_in_box() method

* perform bike rack filtering

* docstring for filter_eval_boxes

* added unittests for filter_eval_boxes()

* update comment

* fix docstring

* can't use Box type because of a circular import. So switched to 'Box'

* can't use Box type because of a circular import. So switched to 'Box'

* change threshold after bikerack filtering

* get max_dist from eval_detection_configs

python-sdk/nuscenes/eval/detection/loaders.py

@@ -3,12 +3,17 @@
 # Licensed under the Creative Commons [see licence.txt]
 
 import json
+from typing import Dict
 
 import numpy as np
 import tqdm
+from pyquaternion import Quaternion
 
+from nuscenes import NuScenes
 from nuscenes.eval.detection.data_classes import EvalBoxes, EvalBox
 from nuscenes.eval.detection.utils import category_to_detection_name
+from nuscenes.utils.geometry_utils import points_in_box
+from nuscenes.utils.data_classes import Box
 from nuscenes.utils.splits import create_splits_scenes
 
 
@@ -105,8 +110,13 @@ def add_center_dist(nusc, eval_boxes: EvalBoxes):
     return eval_boxes
 
 
-def filter_eval_boxes(nusc, eval_boxes: EvalBoxes, max_dist: dict):
-    """ Applies filtering to boxes. Distance, bike-racks and points per box. """
+def filter_eval_boxes(nusc: NuScenes, eval_boxes: EvalBoxes, max_dist: Dict[str, float]) -> EvalBoxes:
+    """
+    Applies filtering to boxes. Distance, bike-racks and points per box.
+    :param nusc: An instance of the NuScenes class.
+    :param eval_boxes: An instance of the EvalBoxes class.
+    :param max_dist: Maps the detection name to the eval distance threshold for that class.
+    """
 
     for sample_token in eval_boxes.sample_tokens:
 
@@ -117,6 +127,22 @@ def filter_eval_boxes(nusc, eval_boxes: EvalBoxes, max_dist: dict):
         # Then remove boxes with zero points in them. Eval boxes have -1 points by default.
         eval_boxes.boxes[sample_token] = [box for box in eval_boxes[sample_token] if not box.num_pts == 0]
 
-        # TODO: add bike-rack filtering
+        # Perform bike-rack filtering
+        sample_anns = nusc.get('sample', sample_token)['anns']
+        bikerack_recs = [nusc.get('sample_annotation', ann) for ann in sample_anns if
+                         nusc.get('sample_annotation', ann)['category_name'] == 'static_object.bicycle_rack']
+
+        filtered_boxes = []
+
+        for rec in bikerack_recs:
+            bikerack_box = Box(rec['translation'], rec['size'], Quaternion(rec['rotation']))
+            for box in eval_boxes[sample_token]:
+                if box.detection_name in ['bicycle', 'motorcycle'] and \
+                        np.sum(points_in_box(bikerack_box, np.expand_dims(np.array(box.translation), axis=1))) > 0:
+                    continue
+                else:
+                    filtered_boxes.append(box)
+
+        eval_boxes.boxes[sample_token] = filtered_boxes
 
     return eval_boxes

python-sdk/nuscenes/eval/detection/tests/test_data_classes.py

@@ -11,7 +11,7 @@
 class TestMetricData(unittest.TestCase):
 
     def test_serialization(self):
-        """ test that instance serialization protocol works with json encodeding """
+        """ test that instance serialization protocol works with json encoding """
         md = MetricData.random_md()
         recovered = MetricData.deserialize(json.loads(json.dumps(md.serialize())))
         self.assertEqual(md, recovered)
@@ -20,7 +20,7 @@ def test_serialization(self):
 class TestMetricDataList(unittest.TestCase):
 
     def test_serialization(self):
-        """ test that instance serialization protocol works with json encodeding """
+        """ test that instance serialization protocol works with json encoding """
         mdl = MetricDataList()
         for i in range(10):
             mdl.set('name', 0.1, MetricData.random_md())
@@ -31,7 +31,7 @@ def test_serialization(self):
 class TestEvalBox(unittest.TestCase):
 
     def test_serialization(self):
-        """ test that instance serialization protocol works with json encodeding """
+        """ test that instance serialization protocol works with json encoding """
         box = EvalBox()
         recovered = EvalBox.deserialize(json.loads(json.dumps(box.serialize())))
         self.assertEqual(box, recovered)
@@ -40,7 +40,7 @@ def test_serialization(self):
 class TestEvalBoxes(unittest.TestCase):
 
     def test_serialization(self):
-        """ test that instance serialization protocol works with json encodeding """
+        """ test that instance serialization protocol works with json encoding """
         boxes = EvalBoxes()
         for i in range(10):
             boxes.add_boxes(str(i), [EvalBox(), EvalBox(), EvalBox()])

python-sdk/nuscenes/eval/detection/tests/test_loader.py

@@ -0,0 +1,124 @@
+# nuScenes dev-kit.
+# Code written by Sourabh Vora, 2019.
+# Licensed under the Creative Commons [see licence.txt]
+
+import os
+import unittest
+
+from nuscenes import NuScenes
+from nuscenes.eval.detection.config import eval_detection_configs
+from nuscenes.eval.detection.loaders import filter_eval_boxes
+from nuscenes.eval.detection.data_classes import EvalBox, EvalBoxes
+
+
+class TestLoader(unittest.TestCase):
+    def test_filter_eval_boxes(self):
+        """
+        This tests runs the evaluation for an arbitrary random set of predictions.
+        This score is then captured in this very test such that if we change the eval code,
+        this test will trigger if the results changed.
+        """
+        assert 'NUSCENES' in os.environ, 'Set NUSCENES env. variable to enable tests.'
+
+        nusc = NuScenes(version='v1.0-mini', dataroot=os.environ['NUSCENES'], verbose=False)
+
+        sample_token = '0af0feb5b1394b928dd13d648de898f5'
+        # This sample has a bike rack instance 'bfe685042aa34ab7b2b2f24ee0f1645f' with these parameters
+        # 'translation': [683.681, 1592.002, 0.809],
+        # 'size': [1.641, 14.465, 1.4],
+        # 'rotation': [0.3473693995546558, 0.0, 0.0, 0.9377283723195315]
+
+        max_dist = eval_detection_configs['cvpr_2019']['class_range']
+
+        # Test bicycle filtering by creating a box at the same position as the bike rack.
+        box1 = EvalBox(sample_token=sample_token,
+                       translation=(683.681, 1592.002, 0.809),
+                       size=(1, 1, 1),
+                       detection_name='bicycle')
+
+        eval_boxes = EvalBoxes()
+        eval_boxes.add_boxes('0af0feb5b1394b928dd13d648de898f5', [box1])
+
+        filtered_boxes = filter_eval_boxes(nusc, eval_boxes, max_dist)
+
+        self.assertEqual(len(filtered_boxes.boxes[sample_token]), 0)    # box1 should be filtered.
+
+        # Test motorcycle filtering by creating a box at the same position as the bike rack.
+        box2 = EvalBox(sample_token=sample_token,
+                       translation=(683.681, 1592.002, 0.809),
+                       size=(1, 1, 1),
+                       detection_name='motorcycle')
+
+        eval_boxes = EvalBoxes()
+        eval_boxes.add_boxes('0af0feb5b1394b928dd13d648de898f5', [box1, box2])
+
+        filtered_boxes = filter_eval_boxes(nusc, eval_boxes, max_dist)
+
+        self.assertEqual(len(filtered_boxes.boxes[sample_token]), 0)    # both box1 and box2 should be filtered.
+
+        # Now create a car at the same position as the bike rack.
+        box3 = EvalBox(sample_token=sample_token,
+                       translation=(683.681, 1592.002, 0.809),
+                       size=(1, 1, 1),
+                       detection_name='car')
+
+        eval_boxes = EvalBoxes()
+        eval_boxes.add_boxes('0af0feb5b1394b928dd13d648de898f5', [box1, box2, box3])
+
+        filtered_boxes = filter_eval_boxes(nusc, eval_boxes, max_dist)
+
+        self.assertEqual(len(filtered_boxes.boxes[sample_token]), 1)  # box1 and box2 to be filtered. box3 to stay.
+        self.assertEqual(filtered_boxes.boxes[sample_token][0].detection_name, 'car')
+
+        # Now add a bike outside the bike rack.
+
+        box4 = EvalBox(sample_token=sample_token,
+                       translation=(68.681, 1592.002, 0.809),
+                       size=(1, 1, 1),
+                       detection_name='bicycle')
+
+        eval_boxes = EvalBoxes()
+        eval_boxes.add_boxes('0af0feb5b1394b928dd13d648de898f5', [box1, box2, box3, box4])
+
+        filtered_boxes = filter_eval_boxes(nusc, eval_boxes, max_dist)
+
+        self.assertEqual(len(filtered_boxes.boxes[sample_token]), 2)  # box1, box2 to be filtered. box3, box4 to stay.
+        self.assertEqual(filtered_boxes.boxes[sample_token][0].detection_name, 'car')
+        self.assertEqual(filtered_boxes.boxes[sample_token][1].detection_name, 'bicycle')
+        self.assertEqual(filtered_boxes.boxes[sample_token][1].translation[0], 68.681)
+
+        # Add another bike on the bike rack center but set the ego_dist higher than what's defined in max_dist
+        box5 = EvalBox(sample_token=sample_token,
+                       translation=(683.681, 1592.002, 0.809),
+                       size=(1, 1, 1),
+                       detection_name='bicycle',
+                       ego_dist=100.0)
+
+        eval_boxes = EvalBoxes()
+        eval_boxes.add_boxes('0af0feb5b1394b928dd13d648de898f5', [box1, box2, box3, box4, box5])
+
+        filtered_boxes = filter_eval_boxes(nusc, eval_boxes, max_dist)
+        self.assertEqual(len(filtered_boxes.boxes[sample_token]), 2)  # box1, box2, box5 filtered. box3, box4 to stay.
+        self.assertEqual(filtered_boxes.boxes[sample_token][0].detection_name, 'car')
+        self.assertEqual(filtered_boxes.boxes[sample_token][1].detection_name, 'bicycle')
+        self.assertEqual(filtered_boxes.boxes[sample_token][1].translation[0], 68.681)
+
+        # Add another bike on the bike rack center but set the num_pts to be zero so that it gets filtered.
+        box6 = EvalBox(sample_token=sample_token,
+                       translation=(683.681, 1592.002, 0.809),
+                       size=(1, 1, 1),
+                       detection_name='bicycle',
+                       num_pts=0)
+
+        eval_boxes = EvalBoxes()
+        eval_boxes.add_boxes('0af0feb5b1394b928dd13d648de898f5', [box1, box2, box3, box4, box5, box6])
+
+        filtered_boxes = filter_eval_boxes(nusc, eval_boxes, max_dist)
+        self.assertEqual(len(filtered_boxes.boxes[sample_token]), 2)  # box1, box2, box5, box6 filtered. box3, box4 stay
+        self.assertEqual(filtered_boxes.boxes[sample_token][0].detection_name, 'car')
+        self.assertEqual(filtered_boxes.boxes[sample_token][1].detection_name, 'bicycle')
+        self.assertEqual(filtered_boxes.boxes[sample_token][1].translation[0], 68.681)
+
+
+if __name__ == '__main__':
+    unittest.main()

python-sdk/nuscenes/eval/detection/tests/test_main.py

@@ -112,7 +112,8 @@ def test_delta(self):
         # 3. Score = 0.24954451673961747. Changed to 1.0-mini and cleaned up build script.
         # 4. Score = 0.20478832626986893. Updated treatment of cones, barriers, and other algo tunings.
         # 5. Score = 0.2043569666105005. AP calculation area is changed from >=min_recall to >min_recall.
-        self.assertAlmostEqual(metrics.weighted_sum, 0.2043569666105005)
+        # 6. Score = 0.20636954644294506. After bike-rack filtering.
+        self.assertAlmostEqual(metrics.weighted_sum, 0.20636954644294506)
 
 
 if __name__ == '__main__':

python-sdk/nuscenes/utils/geometry_utils.py

@@ -2,11 +2,12 @@
 # Code written by Oscar Beijbom, 2018.
 # Licensed under the Creative Commons [see licence.txt]
 
-import numpy as np
 from enum import IntEnum
-from pyquaternion import Quaternion
 from typing import Tuple
 
+import numpy as np
+from pyquaternion import Quaternion
+
 
 class BoxVisibility(IntEnum):
     """ Enumerates the various level of box visibility in an image """
@@ -106,3 +107,40 @@ def transform_matrix(translation: np.ndarray = np.array([0, 0, 0]),
         tm[:3, 3] = np.transpose(np.array(translation))
 
     return tm
+
+
+def points_in_box(box: 'Box', points: float, wlh_factor: float = 1.0):
+    """
+    Checks whether points are inside the box.
+
+    Picks one corner as reference (p1) and computes the vector to a target point (v).
+    Then for each of the 3 axes, project v onto the axis and compare the length.
+    Inspired by: https://math.stackexchange.com/a/1552579
+    :param box: <Box>.
+    :param points: <np.float: 3, n>.
+    :param wlh_factor: Inflates or deflates the box.
+    :return: <np.bool: n, >.
+    """
+    corners = box.corners(wlh_factor=wlh_factor)
+
+    p1 = corners[:, 0]
+    p_x = corners[:, 4]
+    p_y = corners[:, 1]
+    p_z = corners[:, 3]
+
+    i = p_x - p1
+    j = p_y - p1
+    k = p_z - p1
+
+    v = points - p1.reshape((-1, 1))
+
+    iv = np.dot(i, v)
+    jv = np.dot(j, v)
+    kv = np.dot(k, v)
+
+    mask_x = np.logical_and(0 <= iv, iv <= np.dot(i, i))
+    mask_y = np.logical_and(0 <= jv, jv <= np.dot(j, j))
+    mask_z = np.logical_and(0 <= kv, kv <= np.dot(k, k))
+    mask = np.logical_and(np.logical_and(mask_x, mask_y), mask_z)
+
+    return mask

python-sdk/nuscenes/utils/tests/test_geometry_utils.py

@@ -8,6 +8,8 @@
 from pyquaternion import Quaternion
 
 from nuscenes.eval.detection.utils import quaternion_yaw
+from nuscenes.utils.data_classes import Box
+from nuscenes.utils.geometry_utils import points_in_box
 
 
 class TestGeometryUtils(unittest.TestCase):
@@ -54,6 +56,61 @@ def test_quaternion_yaw(self):
         yaw_test = quaternion_yaw(q)
         self.assertAlmostEqual(yaw_in, yaw_test)
 
+    def test_points_in_box(self):
+        """ Test the box.in_box method. """
+
+        vel = (np.nan, np.nan, np.nan)
+
+        def qyaw(yaw):
+            return Quaternion(axis=(0, 0, 1), angle=yaw)
+
+        # Check points inside box
+        box = Box([0.0, 0.0, 0.0], [2.0, 2.0, 0.0], qyaw(0.0), 1, 2.0, vel)
+        points = np.array([[0.0, 0.0, 0.0], [0.5, 0.5, 0.0]]).transpose()
+        mask = points_in_box(box, points, wlh_factor=1.0)
+        self.assertEqual(mask.all(), True)
+
+        # Check points outside box
+        box = Box([0.0, 0.0, 0.0], [2.0, 2.0, 0.0], qyaw(0.0), 1, 2.0, vel)
+        points = np.array([[0.1, 0.0, 0.0], [0.5, -1.1, 0.0]]).transpose()
+        mask = points_in_box(box, points, wlh_factor=1.0)
+        self.assertEqual(mask.all(), False)
+
+        # Check corner cases
+        box = Box([0.0, 0.0, 0.0], [2.0, 2.0, 0.0], qyaw(0.0), 1, 2.0, vel)
+        points = np.array([[-1.0, -1.0, 0.0], [1.0, 1.0, 0.0]]).transpose()
+        mask = points_in_box(box, points, wlh_factor=1.0)
+        self.assertEqual(mask.all(), True)
+
+        # Check rotation (45 degs) and translation (by [1,1])
+        rot = 45
+        trans = [1.0, 1.0]
+        box = Box([0.0+trans[0], 0.0+trans[1], 0.0], [2.0, 2.0, 0.0], qyaw(rot / 180.0 * np.pi), 1, 2.0, vel)
+        points = np.array([[0.70+trans[0], 0.70+trans[1], 0.0], [0.71+1.0, 0.71+1.0, 0.0]]).transpose()
+        mask = points_in_box(box, points, wlh_factor=1.0)
+        self.assertEqual(mask[0], True)
+        self.assertEqual(mask[1], False)
+
+        # Check 3d box
+        box = Box([0.0, 0.0, 0.0], [2.0, 2.0, 2.0], qyaw(0.0), 1, 2.0, vel)
+        points = np.array([[0.0, 0.0, 0.0], [0.5, 0.5, 0.5]]).transpose()
+        mask = points_in_box(box, points, wlh_factor=1.0)
+        self.assertEqual(mask.all(), True)
+
+        # Check wlh factor
+        for wlh_factor in [0.5, 1.0, 1.5, 10.0]:
+            box = Box([0.0, 0.0, 0.0], [2.0, 2.0, 0.0], qyaw(0.0), 1, 2.0, vel)
+            points = np.array([[0.0, 0.0, 0.0], [0.5, 0.5, 0.0]]).transpose()
+            mask = points_in_box(box, points, wlh_factor=wlh_factor)
+            self.assertEqual(mask.all(), True)
+
+        for wlh_factor in [0.1, 0.49]:
+            box = Box([0.0, 0.0, 0.0], [2.0, 2.0, 0.0], qyaw(0.0), 1, 2.0, vel)
+            points = np.array([[0.0, 0.0, 0.0], [0.5, 0.5, 0.0]]).transpose()
+            mask = points_in_box(box, points, wlh_factor=wlh_factor)
+            self.assertEqual(mask[0], True)
+            self.assertEqual(mask[1], False)
+
 
 if __name__ == '__main__':
     unittest.main()