add cam_instances

check_coor.py

@@ -0,0 +1,24 @@
+import mmengine
+import numpy as np
+
+from mmdet3d.structures import (Box3DMode, CameraInstance3DBoxes,
+                                LiDARInstance3DBoxes)
+
+val_infos = mmengine.load('data/waymo_mini/kitti_format/waymo_infos_val.pkl')
+old_val_infos = mmengine.load(
+    'data/waymo_mini/kitti_format/old_pkl/waymo_infos_val.pkl')
+instance = np.array(
+    val_infos['data_list'][0]['cam_sync_instances'][0]['bbox_3d'])[np.newaxis,
+                                                                   ...]
+instance = LiDARInstance3DBoxes(instance)
+lidar2cam = np.array(
+    val_infos['data_list'][0]['images']['CAM_FRONT']['lidar2cam'])
+cam_instance = instance.convert_to(Box3DMode.CAM, lidar2cam)
+
+old_instance = np.array(old_val_infos['data_list'][0]['cam_sync_instances'][0]
+                        ['bbox_3d'])[np.newaxis, ...]
+
+old_lidar2cam = np.array(
+    old_val_infos['data_list'][0]['images']['CAM_FRONT']['lidar2cam'])
+old_instance = CameraInstance3DBoxes(old_instance)
+pass

get_gt_mini.py

@@ -0,0 +1,11 @@
+from waymo_open_dataset.protos.metrics_pb2 import Objects
+
+objects = Objects()
+with open('./data/waymo_mini/waymo_format/gt.bin', 'rb') as f:
+    objects.ParseFromString(bytearray(f.read()))
+new = Objects()
+for obj in objects.objects:
+    if obj.context_name == '10203656353524179475_7625_000_7645_000':
+        new.objects.append(obj)
+with open('./data/waymo_mini/waymo_format/gt_mini.bin', 'wb') as f:
+    f.write(new.SerializeToString())

mmdet3d/datasets/convert_utils.py

@@ -319,7 +319,11 @@ def convert_annos(info: dict, cam_idx: int) -> dict:
     """Convert front-cam anns to i-th camera (KITTI-style info)."""
     rect = info['calib']['R0_rect'].astype(np.float32)
     # lidar2cam0 = info['calib']['Tr_velo_to_cam'].astype(np.float32)
-    lidar2cami = info['calib'][f'Tr_velo_to_cam{cam_idx}'].astype(np.float32)
+    if cam_idx == 0:
+        lidar2cami = info['calib']['Tr_velo_to_cam'].astype(np.float32)
+    else:
+        lidar2cami = info['calib'][f'Tr_velo_to_cam{cam_idx}'].astype(
+            np.float32)
     annos = info['annos']
     converted_annos = copy.deepcopy(annos)
     loc = annos['location']
@@ -336,7 +340,7 @@ def convert_annos(info: dict, cam_idx: int) -> dict:
     # gt_bboxes_3d = gt_bboxes_3d.convert_to(
     #     Box3DMode.CAM, rect @ lidar2cami, correct_yaw=True).numpy()
     gt_bboxes_3d = LiDARInstance3DBoxes(gt_bboxes_3d).convert_to(
-        Box3DMode.CAM, rect @ lidar2cami, correct_yaw=True)
+        Box3DMode.CAM, rect @ lidar2cami, correct_yaw=True).numpy()
 
     converted_annos['location'] = gt_bboxes_3d[:, :3]
     converted_annos['dimensions'] = gt_bboxes_3d[:, 3:6]

projects/DSVT/configs/dsvt_voxel032_res-second_secfpn_8xb1-cyclic-12e_waymoD5-3d-3class.py

@@ -5,8 +5,8 @@
 voxel_size = [0.32, 0.32, 6]
 grid_size = [468, 468, 1]
 point_cloud_range = [-74.88, -74.88, -2, 74.88, 74.88, 4.0]
-# data_root = 'data/waymo_mini/kitti_format/'
-data_root = 'data/waymo/kitti_format/'
+data_root = 'data/waymo_mini/kitti_format/'
+# data_root = 'data/waymo/kitti_format/'
 class_names = ['Car', 'Pedestrian', 'Cyclist']
 metainfo = dict(classes=class_names)
 input_modality = dict(use_lidar=True, use_camera=False)
@@ -193,7 +193,7 @@
 
 dataset_type = 'WaymoDataset'
 val_dataloader = dict(
-    batch_size=4,
+    batch_size=1,
     num_workers=4,
     persistent_workers=True,
     drop_last=False,
@@ -211,22 +211,22 @@
         backend_args=backend_args))
 test_dataloader = val_dataloader
 
-# val_evaluator = dict(
-#     type='WaymoMetric',
-#     ann_file='./data/waymo_mini/kitti_format/waymo_infos_val.pkl',
-#     waymo_bin_file='./data/waymo_mini/waymo_format/gt_mini.bin',
-#     data_root='./data/waymo_mini/waymo_format',
-#     backend_args=backend_args,
-#     convert_kitti_format=False,
-#     idx2metainfo='./data/waymo_mini/waymo_format/idx2metainfo.pkl')
 val_evaluator = dict(
     type='WaymoMetric',
-    ann_file='./data/waymo/kitti_format/waymo_infos_val.pkl',
-    waymo_bin_file='./data/waymo/waymo_format/gt.bin',
-    data_root='./data/waymo/waymo_format',
+    ann_file='./data/waymo_mini/kitti_format/waymo_infos_val.pkl',
+    waymo_bin_file='./data/waymo_mini/waymo_format/gt_mini.bin',
+    data_root='./data/waymo_mini/waymo_format',
     backend_args=backend_args,
     convert_kitti_format=False,
-    idx2metainfo='./data/waymo/waymo_format/idx2metainfo.pkl')
+    idx2metainfo='./data/waymo_mini/waymo_format/idx2metainfo.pkl')
+# val_evaluator = dict(
+#     type='WaymoMetric',
+#     ann_file='./data/waymo/kitti_format/waymo_infos_val.pkl',
+#     waymo_bin_file='./data/waymo/waymo_format/gt.bin',
+#     data_root='./data/waymo/waymo_format',
+#     backend_args=backend_args,
+#     convert_kitti_format=False,
+#     idx2metainfo='./data/waymo/waymo_format/idx2metainfo.pkl')
 test_evaluator = val_evaluator
 
 vis_backends = [dict(type='LocalVisBackend')]

tools/create_data.py

@@ -173,7 +173,7 @@ def waymo_data_prep(root_path,
                     version,
                     out_dir,
                     workers,
-                    max_sweeps=5):
+                    max_sweeps=10):
     """Prepare the info file for waymo dataset.
 
     Args:
@@ -275,11 +275,16 @@ def semantickitti_data_prep(info_prefix, out_dir):
     '--only-gt-database',
     action='store_true',
     help='Whether to only generate ground truth database.')
+parser.add_argument(
+    '--skip-extract-kitti-format',
+    action='store_true',
+    help='''Whether to skip extracting kitti format image and point cloud.
+    Only used when dataset is waymo!''')
 args = parser.parse_args()
 
 if __name__ == '__main__':
-    from mmdet3d.utils import register_all_modules
-    register_all_modules()
+    from mmengine.registry import init_default_scope
+    init_default_scope('mmdet3d')
 
     if args.dataset == 'kitti':
         if args.only_gt_database:
@@ -334,6 +339,22 @@ def semantickitti_data_prep(info_prefix, out_dir):
                 dataset_name='NuScenesDataset',
                 out_dir=args.out_dir,
                 max_sweeps=args.max_sweeps)
+    elif args.dataset == 'waymo' and args.version == 'v1.4':
+        waymo_data_prep(
+            root_path=args.root_path,
+            info_prefix=args.extra_tag,
+            version=args.version,
+            out_dir=args.out_dir,
+            workers=args.workers,
+            max_sweeps=args.max_sweeps)
+    elif args.dataset == 'waymo' and args.version == 'v1.4-mini':
+        waymo_data_prep(
+            root_path=args.root_path,
+            info_prefix=args.extra_tag,
+            version=args.version,
+            out_dir=args.out_dir,
+            workers=args.workers,
+            max_sweeps=args.max_sweeps)
     elif args.dataset == 'lyft':
         train_version = f'{args.version}-train'
         lyft_data_prep(
@@ -347,14 +368,6 @@ def semantickitti_data_prep(info_prefix, out_dir):
             info_prefix=args.extra_tag,
             version=test_version,
             max_sweeps=args.max_sweeps)
-    elif args.dataset == 'waymo':
-        waymo_data_prep(
-            root_path=args.root_path,
-            info_prefix=args.extra_tag,
-            version=args.version,
-            out_dir=args.out_dir,
-            workers=args.workers,
-            max_sweeps=args.max_sweeps)
     elif args.dataset == 'scannet':
         scannet_data_prep(
             root_path=args.root_path,

tools/dataset_converters/update_infos_to_v2.py

@@ -961,18 +961,22 @@ def update_waymo_infos(pkl_path, out_dir):
             lidar_sweep['timestamp'] = ori_sweep['timestamp']
             lidar_sweep['lidar_points']['lidar_path'] = Path(
                 ori_sweep['velodyne_path']).name
+            # delete the num_pts_feats and lidar2ego
+            del lidar_sweep['lidar_points']['num_pts_feats']
+            del lidar_sweep['lidar_points']['lidar2ego']
+            # TODO: refactor image_sweeps info when we really need it.
             # image sweeps
             # image_sweep = dict()
             # image_sweep['sample_idx']= ori_sweep['image']['image_idx']
-            image_sweep = get_single_image_sweep(camera_types)
-            image_sweep['ego2global'] = ori_sweep['pose']
-            image_sweep['timestamp'] = ori_sweep['timestamp']
-            img_path = Path(ori_sweep['image_path']).name
-            for cam_idx, cam_key in enumerate(camera_types):
-                image_sweep['images'][cam_key]['img_path'] = img_path
+            # image_sweep = get_single_image_sweep(camera_types)
+            # image_sweep['ego2global'] = ori_sweep['pose']
+            # image_sweep['timestamp'] = ori_sweep['timestamp']
+            # img_path = Path(ori_sweep['image_path']).name
+            # for cam_idx, cam_key in enumerate(camera_types):
+            #     image_sweep['images'][cam_key]['img_path'] = img_path
 
             temp_data_info['lidar_sweeps'].append(lidar_sweep)
-            temp_data_info['image_sweeps'].append(image_sweep)
+            # temp_data_info['image_sweeps'].append(image_sweep)
 
         anns = ori_info_dict.get('annos', None)
         ignore_class_name = set()
@@ -1107,8 +1111,8 @@ def generate_waymo_camera_instances(ori_info_dict, cam_keys):
 
     for cam_idx, cam_key in enumerate(cam_keys):
         annos = copy.deepcopy(ori_info_dict['cam_sync_annos'])
-        if cam_idx != 0:
-            annos = convert_annos(ori_info_dict, cam_idx)
+
+        annos = convert_annos(ori_info_dict, cam_idx)
 
         ann_infos = get_kitti_style_2d_boxes(
             ori_info_dict, cam_idx, occluded=[0], annos=annos, dataset='waymo')

tools/dataset_converters/waymo_converter.py

@@ -92,15 +92,15 @@ def __init__(self,
         self.tfrecord_pathnames = sorted(
             glob(join(self.load_dir, '*.tfrecord')))
 
-        self.label_save_dir = f'{self.save_dir}/label_'
+        # self.label_save_dir = f'{self.save_dir}/label_'
         self.label_all_save_dir = f'{self.save_dir}/label_all'
         self.image_save_dir = f'{self.save_dir}/image_'
         self.calib_save_dir = f'{self.save_dir}/calib'
         self.point_cloud_save_dir = f'{self.save_dir}/velodyne'
         self.pose_save_dir = f'{self.save_dir}/pose'
         self.timestamp_save_dir = f'{self.save_dir}/timestamp'
         if self.save_cam_sync_labels:
-            self.cam_sync_label_save_dir = f'{self.save_dir}/cam_sync_label_'
+            # self.cam_sync_label_save_dir = f'{self.save_dir}/cam_sync_label_'
             self.cam_sync_label_all_save_dir = \
                 f'{self.save_dir}/cam_sync_label_all'
 
@@ -450,10 +450,10 @@ def create_folder(self):
                 self.pose_save_dir,
                 self.timestamp_save_dir,
             ]
-            dir_list2 = [self.label_save_dir, self.image_save_dir]
+            dir_list2 = [self.image_save_dir]
             if self.save_cam_sync_labels:
                 dir_list1.append(self.cam_sync_label_all_save_dir)
-                dir_list2.append(self.cam_sync_label_save_dir)
+                # dir_list2.append(self.cam_sync_label_save_dir)
         else:
             dir_list1 = [
                 self.calib_save_dir, self.pose_save_dir,