Changes of filter for DSM

.hydra/config.yaml

@@ -1,27 +1,23 @@
 work_dir: ${hydra:runtime.cwd}
 data_dir: ${work_dir}/data
 print_config: true
-type: inference
-task_name: demo
-num_leaf: 8
-suffix: ''
-save_demo: false
-save_wis3d: false
-use_tracking: false
-model:
-  onepose_model_path: ${work_dir}/data/models/checkpoints/onepose/GATsSPG.ckpt
-  extractor_model_path: ${work_dir}/data/models/extractors/SuperPoint/superpoint_v1.pth
-  match_model_path: ${work_dir}/data/models/matchers/SuperGlue/superglue_outdoor.pth
-scan_data_dir: ${data_dir}/onepose_datasets/val_data
-sfm_model_dir: ${data_dir}/sfm_model
+type: sfm
+redo: true
+hololens: false
+scan_data_dir: ${work_dir}/data/onepose_datasets/val_data
+dataset:
+  max_num_kp3d: 2500
+  max_num_kp2d: 1000
+  data_dir:
+  - ${scan_data_dir}/0620-dinosaurcup-bottle dinosaurcup-1
+  outputs_dir: ${work_dir}/data/sfm_model/{}
 network:
   detection: superpoint
+  detection_model_path: ${work_dir}/data/models/extractors/SuperPoint/superpoint_v1.pth
   matching: superglue
-max_num_kp3d: 2500
-input:
-  data_dirs: /Users/PELLERITO/Desktop/mixed_reality_code/OnePose/data/onepose_datasets/tiger
-    tiger-test
-  sfm_model_dirs: /Users/PELLERITO/Desktop/mixed_reality_code/OnePose/data/sfm_model/tiger
-output:
-  vis_dir: ${work_dir}/runs/vis/demo
-  eval_dir: ${work_dir}/runs/eval/demo
+  matching_model_path: ${work_dir}/data/models/matchers/SuperGlue/superglue_outdoor.pth
+sfm:
+  down_ratio: 5
+  covis_num: 10
+  rotation_thresh: 50
+disable_lightning_logs: true

.hydra/hydra.yaml

@@ -109,15 +109,10 @@ hydra:
   overrides:
     hydra: []
     task:
-    - +experiment=test_demo
-    - use_tracking=False
-    - input.data_dirs=/Users/PELLERITO/Desktop/mixed_reality_code/OnePose/data/onepose_datasets/tiger
-      tiger-test
-    - input.sfm_model_dirs=/Users/PELLERITO/Desktop/mixed_reality_code/OnePose/data/sfm_model/tiger
+    - +preprocess=sfm_spp_spg_val.yaml
   job:
-    name: inference_demo
-    override_dirname: +experiment=test_demo,input.data_dirs=/Users/PELLERITO/Desktop/mixed_reality_code/OnePose/data/onepose_datasets/tiger
-      tiger-test,input.sfm_model_dirs=/Users/PELLERITO/Desktop/mixed_reality_code/OnePose/data/sfm_model/tiger,use_tracking=False
+    name: run
+    override_dirname: +preprocess=sfm_spp_spg_val.yaml
     id: ???
     num: ???
     config_name: config.yaml
@@ -130,19 +125,19 @@ hydra:
         exclude_keys: []
   runtime:
     version: 1.1.1
-    cwd: /Users/PELLERITO/Desktop/mixed_reality_code/OnePose
+    cwd: /Users/diego/Desktop/Escritorio_MacBook_Pro_de_Diego/ETH/Third_Semester/Mixed_Reality/Real_2/ZeroShotPoseEstimation
     config_sources:
     - path: hydra.conf
       schema: pkg
       provider: hydra
-    - path: /Users/PELLERITO/Desktop/mixed_reality_code/OnePose/configs
+    - path: /Users/diego/Desktop/Escritorio_MacBook_Pro_de_Diego/ETH/Third_Semester/Mixed_Reality/Real_2/ZeroShotPoseEstimation/configs
       schema: file
       provider: main
     - path: ''
       schema: structured
       provider: schema
     choices:
-      experiment: test_demo
+      preprocess: sfm_spp_spg_val.yaml
       logger: null
       callbacks: null
       datamodule: null

.hydra/overrides.yaml

@@ -1,5 +1 @@
-- +experiment=test_demo
-- use_tracking=False
-- input.data_dirs=/Users/PELLERITO/Desktop/mixed_reality_code/OnePose/data/onepose_datasets/tiger
-  tiger-test
-- input.sfm_model_dirs=/Users/PELLERITO/Desktop/mixed_reality_code/OnePose/data/sfm_model/tiger
+- +preprocess=sfm_spp_spg_val.yaml

configs/preprocess/sfm_spp_spg_val.yaml

@@ -3,18 +3,17 @@
 type: sfm
 work_dir: ${hydra:runtime.cwd}
 redo: True
-hololens: True
+hololens: False
 
 # scan_data_dir: ${work_dir}/data/onepose_datasets/val_data
-scan_data_dir: ${work_dir}/data/costum_datasets/test
-
+scan_data_dir: ${work_dir}/data/onepose_datasets/val_data
 dataset:
     max_num_kp3d: 2500
     max_num_kp2d: 1000
 
     data_dir:
         # - ${scan_data_dir}/0606-tiger-others tiger-2
-        - ${scan_data_dir}/demo_capture demo-1
+        - ${scan_data_dir}/0620-dinosaurcup-bottle dinosaurcup-1
 
     outputs_dir: ${work_dir}/data/sfm_model/{}
 

feature_matching_object_detector.py

@@ -149,9 +149,6 @@ def inference_core(cfg, data_root, seq_dir, sfm_model_dir):
 
     # Begin Object detection:
     for id, data in enumerate(tqdm(loader)):
-    #for data in (tqdm(loader)):
-
-    #for id, data in enumerate((loader)):
 
         img_path = data["path"][0]
         inp = data["image"].to(device)

run.py

@@ -121,11 +121,12 @@ def sfm(cfg):
         for sub_dir in sub_dirs:
             seq_dir = osp.join(root_dir, sub_dir)
             img_paths += glob.glob(str(Path(seq_dir)) + "/color/*.png", recursive=True)
-            full_res_img_paths += glob.glob(str(Path(seq_dir)) + "/color_full/*.png", recursive=True)
-            poses_paths += glob.glob(str(Path(seq_dir)) + "/poses/*.txt", recursive=True)
-            intrinsics_path = str(Path(seq_dir)) + "/intrinsics.txt"
+            full_res_img_paths += glob.glob(str(Path(seq_dir)) + "/color_full/*.jpg", recursive=True)
             poses_paths += glob.glob(str(Path(seq_dir)) + "/poses/*.txt", recursive=True)
             intrinsics_path = str(Path(seq_dir)) + "/intrinsics.txt"
+
+            #poses_paths += glob.glob(str(Path(seq_dir)) + "/poses/*.txt", recursive=True)
+            #intrinsics_path = str(Path(seq_dir)) + "/intrinsics.txt"
 
         # Choose less images from the list to build the sfm model
 
@@ -149,14 +150,20 @@ def sfm(cfg):
             poses_paths=poses_paths,
             data_root=root_dir,
             seq_dir = seq_dir,
-            compute_on_GPU="cuda",
+            compute_on_GPU="cpu",
             step=1,
-            hololens=cfg.hololens
+            hololens=cfg.hololens,
+            filter = False, #DSM Filter
         )
 
+
         # Begin SfM and postprocess:
+        print("Beginning of SFM")
         sfm_core(cfg, down_img_lists, outputs_dir_root)
+        print("SfM done")
+        print("Beginning of Post-process")
         postprocess(cfg, down_img_lists, root_dir, outputs_dir_root)
+        print("Post-process done")
 
 
 def sfm_core(cfg, img_lists, outputs_dir_root):
@@ -180,6 +187,7 @@ def sfm_core(cfg, img_lists, outputs_dir_root):
     empty_dir = osp.join(outputs_dir, "sfm_empty")
     deep_sfm_dir = osp.join(outputs_dir, "sfm_ws")
 
+
     if cfg.redo:
         os.system(f"rm -rf {outputs_dir}")
         Path(outputs_dir).mkdir(exist_ok=True, parents=True)

src/bbox_3D_estimation/utils.py

@@ -41,6 +41,7 @@ def detect_3D_box(self):
         estQs = compute_estimates(self.bboxes, self.K, self.poses, self.visibility)
         centre, axes, R = dual_quadric_to_ellipsoid_parameters(estQs[object_idx])
 
+        print("Axes = ", axes)
         # Possible coordinates
         mins = [-ax for (ax) in axes]
         maxs = [ax for (ax) in axes]
@@ -58,31 +59,17 @@ def detect_3D_box(self):
         self.points = points
         self.centre = centre
         self.R = R
+        self.estQs = estQs
 
         # Transformation to have coordinates centered in the bounding box (and aligned with it)
         M = np.empty((4, 4))
         M[:3, :3] = R
         M[:3, 3] = centre
         M[3, :] = [0, 0, 0, 1]
-        # print(M)
 
         self.M = np.linalg.inv(M)
 
 
-        # gt_p = np.loadtxt(f"data/onepose_datasets/val_data/0606-tiger-others/box3d_corners_GT.txt")
-
-        # plot_3D_scene(
-        # estQs=estQs,
-        # gtQs=gt_p,
-        # Ms_t=self.poses,
-        # dataset="tiger",
-        # save_output_images=False,
-        # points=points,
-        # GT_points=gt_p 
-        # )
-        # plt.show()
-
-
     def save_3D_box(self, data_root):
         np.savetxt(data_root + "/box3d_corners.txt", self.points, delimiter=" ")
 
@@ -105,33 +92,73 @@ def save_poses(self, seq_dir):
     def save_dimensions(self, data_root):
         np.savetxt(data_root + "/box3d_dimensions.txt", self.axes, delimiter=" ")
 
+    def plot_3D_bb(self, poses, GT_points=None):
+        plot_3D_scene(
+            estQs=self.estQs,
+            gtQs=None,
+            Ms_t=poses,
+            dataset="tiger",
+            save_output_images=True,
+            points=self.points,
+            GT_points=GT_points 
+        )
+        plt.show()
+
 
 def predict_3D_bboxes(
     full_res_img_paths,
     intrisics_path,
     poses_paths,
     data_root,
     seq_dir,
+    filter,
     step=1,
-    downscale_factor=0.3,
+    downscale_factor=0.6,
     compute_on_GPU="cpu",
     hololens=False
+
 ):  
     full_res_img_paths = sort_path_list(full_res_img_paths)
     poses_paths = sort_path_list(poses_paths)
     _K, _ = data_utils.get_K(intrisics_path) 
 
     DetectorBox3D = Detector3D(_K)
-    BboxPredictor = UnsupBbox(downscale_factor=downscale_factor, device=compute_on_GPU)
+    BboxPredictor = UnsupBbox(downscale_factor=downscale_factor, device=compute_on_GPU, filter = filter)
+
 
     for id, img_path in enumerate(tqdm(full_res_img_paths)):
         if id % step == 0 or id == 0:
+
             image = cv2.imread(str(img_path))
             image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
+
             poses = read_list_poses([poses_paths[id]], hololens=hololens)
             poses_orig = read_list_poses_orig([poses_paths[id]])
-
             bbox_orig_res = BboxPredictor.infer_2d_bbox(image=image, K=_K)
+
+
+            '''
+            ##--------Plotting the RGB image---------
+            image_2 = image.copy()
+            limits = bbox_orig_res
+            start_point = (int(limits[0]), int(limits[1]))
+            end_point = (int(limits[2]), int(limits[3]))
+
+            # Blue color in BGR
+            color = (255, 0, 0)            
+            # Line thickness of 2 px
+            thickness = 2
+
+            # Draw a rectangle with blue line borders of thickness of 2 px
+            image_2 = cv2.rectangle(image_2, start_point, end_point, color, thickness)
+
+            cv2.imshow("Image", image_2)
+            #cv2.imwrite('/Users/diego/Desktop/Escritorio_MacBook_Pro_de_Diego/ETH/Third_Semester/Mixed_Reality/Real_2/ZeroShotPoseEstimation/data/onepose_datasets/val_data/0620-dinosaurcup-bottle/dinosaurcup-1/detection_2/'+str(id)+ '.jpg', image_2)
+            cv2.waitKey(1)
+            '''
+
+
+            #1 is for the bbox without a filter, 0 is with filter
             DetectorBox3D.add_view(bbox_orig_res, poses, poses_orig)
 
     DetectorBox3D.detect_3D_box()
@@ -140,6 +167,20 @@ def predict_3D_bboxes(
     DetectorBox3D.save_poses(seq_dir)
     DetectorBox3D.save_dimensions(data_root)
 
+    poses_t=None
+    for id, img_path in enumerate(tqdm(full_res_img_paths)):
+        poses = read_list_poses([poses_paths[id]], hololens=False)
+        if poses_t is None:
+            poses_t = poses
+        else:
+            poses_t = np.vstack((poses_t, poses))
+
+    #DetectorBox3D.plot_3D_bb(poses_t)
+    #print("-------Predict 3D bboxes finished-------")
+
+    GT_points = np.loadtxt(data_root + "/box3d_corners_GT.txt")
+
+    DetectorBox3D.plot_3D_bb(poses_t, GT_points=GT_points)
 
 def sort_path_list(path_list):
     files = {int(Path(file).stem): file for file in path_list}
@@ -477,6 +518,7 @@ def estimate_one_ellipsoid(Ps_t, Cs):
             i += 1
             print(i)
 
+
     _, _, V = np.linalg.svd(M)
     w = V[-1, :]  # V is transposed respect to Matlab, so we take the last row.