Merge remote-tracking branch 'origin/master'

Author: Dan Jia

README.md

@@ -1,10 +1,14 @@
 # Person Detection in 2D Range Data
-This repository implements DROW3 [(arXiv)](https://arxiv.org/abs/1804.02463) and DR-SPAAM [(arXiv)](https://arxiv.org/abs/2004.14079), real-time person detectors using 2D LiDARs mounted at ankle or knee height.
-Also included are experiments from *Self-Supervised Person Detection in 2D Range Data using a Calibrated Camera* [(arXiv)](https://arxiv.org/abs/2012.08890).
+This repository implements DROW3 ([arXiv](https://arxiv.org/abs/1804.02463)) and DR-SPAAM ([arXiv](https://arxiv.org/abs/2004.14079)), real-time person detectors using 2D LiDARs mounted at ankle or knee height.
+Also included are experiments from *Self-Supervised Person Detection in 2D Range Data using a Calibrated Camera* ([arXiv](https://arxiv.org/abs/2012.08890)).
 Pre-trained models (using PyTorch 1.6) can be found in this [Google drive](https://drive.google.com/drive/folders/1Wl2nC8lJ6s9NI1xtWwmxeAUnuxDiiM4W?usp=sharing).
 
 ![](imgs/teaser_1.gif)
 
+## News
+
+[06-03-2021] Our work has been accepted to ICRA'21! Checkout the presentation video [here](https://www.youtube.com/watch?v=f5U1ZfqXtc0).
+
 ## Quick start
 
 First clone and install the repository
@@ -114,7 +118,7 @@ If your robot has a calibrated camera (i.e. the transformation between the camer
 you can generate pseudo labels automatically during deployment and fine-tune the detector (no manual labeling needed).
 We provide a wrapper function `dr_spaam.pseudo_labels.get_regression_target_using_bounding_boxes()` for generating pseudo labels conveniently.
 For experiments using pseudo labels,
-checkout our paper *Self-Supervised Person Detection in 2D Range Data using a Calibrated Camera* [(arXiv)](https://arxiv.org/abs/2012.08890).
+checkout our paper *Self-Supervised Person Detection in 2D Range Data using a Calibrated Camera* ([arXiv](https://arxiv.org/abs/2012.08890)).
 Use checkpoints in this [Google drive](https://drive.google.com/drive/folders/1Wl2nC8lJ6s9NI1xtWwmxeAUnuxDiiM4W?usp=sharing) to reproduce our results.
 
 ## Inference time

dr_spaam/bin/analyze_pseudo_labels.py renamed to dr_spaam/bin/plotting/analyze_pseudo_labels.py

@@ -8,7 +8,7 @@
 import matplotlib.pyplot as plt
 from matplotlib.gridspec import GridSpec
 
-from dr_spaam.dataset.get_dataloader import get_dataloader
+from dr_spaam.dataset import get_dataloader
 import dr_spaam.utils.jrdb_transforms as jt
 import dr_spaam.utils.precision_recall as pru
 import dr_spaam.utils.utils as u
@@ -254,7 +254,7 @@ def _write_file_make_dir(f_name, f_str):
 
 
 def generate_pseudo_labels():
-    with open("./base_dr_spaam_jrdb_cfg.yaml", "r") as f:
+    with open("./cfgs/base_dr_spaam_jrdb_cfg.yaml", "r") as f:
         cfg = yaml.safe_load(f)
     cfg["dataset"]["pseudo_label"] = True
     cfg["dataset"]["pl_correction_level"] = 0

dr_spaam/bin/plotting/get_eer_thresh.py

@@ -0,0 +1,13 @@
+import pickle
+import numpy as np
+
+p1 = "/home/jia/git/awesome_repos/2D_lidar_person_detection/dr_spaam/logs/20210520_224024_drow_jrdb_EVAL/output/val/e000000/evaluation/all/result_r05.pkl"
+p2 = "/home/jia/git/awesome_repos/2D_lidar_person_detection/dr_spaam/logs/20210520_231344_drow_jrdb_EVAL/output/val/e000000/evaluation/all/result_r05.pkl"
+
+for p in (p1, p2):
+    with open(p, "rb") as f:
+        res = pickle.load(f)
+
+    eer = res["eer"]
+    arg = np.argmin(np.abs(res["precisions"] - eer))
+    print(res["thresholds"][arg], " ", res["precisions"][arg], " ", res["recalls"][arg])

dr_spaam/bin/get_pseudo_label_videos.py renamed to dr_spaam/bin/plotting/get_pseudo_label_videos.py

@@ -7,19 +7,25 @@
 
 import torch
 
-from dr_spaam.dataset.get_dataloader import get_dataloader
+from dr_spaam.dataset import get_dataloader
 import dr_spaam.utils.jrdb_transforms as jt
 import dr_spaam.utils.utils as u
 
 from dr_spaam.pipeline.logger import Logger
 from dr_spaam.model.get_model import get_model
 
+# for _PLOTTING_INTERVAL = 2
+# 000579 = 14s
+# ffmpeg -r 20 -pattern_type glob -i 'packard-poster-session-2019-03-20_1/scan_pl_*.png' -c:v libx264 -vf fps=30 -pix_fmt yuv420p out_pl.mp4
+
 _X_LIM = (-15, 15)
 # _Y_LIM = (-10, 4)
 _Y_LIM = (-7, 7)
 
 _PLOTTING_INTERVAL = 2
 _MAX_COUNT = 1e9
+# _MAX_COUNT = 1e1
+_SEQ_MAX_COUNT = 2000
 
 # _COLOR_CLOSE_HSV = (1.0, 0.59, 0.75)
 _COLOR_CLOSE_HSV = (0.0, 1.0, 1.0)
@@ -52,7 +58,7 @@ def _distance_to_bgr_color(dist):
     return c_bgr[0]
 
 
-def _plot_frame_im(batch_dict, ib):
+def _plot_frame_im(batch_dict, ib, show_pseudo_labels=False):
     frame_id = f"{batch_dict['frame_id'][ib]:06d}"
     sequence = batch_dict["sequence"][ib]
 
@@ -82,50 +88,65 @@ def _plot_frame_im(batch_dict, ib):
     scan_xyz_laser = np.stack((scan_x, -scan_y, scan_z), axis=0)  # in JRDB laser frame
     p_xy, ib_mask = jt.transform_pts_laser_to_stitched_im(scan_xyz_laser)
 
-    # detection bounding box
-    for box_dict in batch_dict["im_dets"][ib]:
-        x0, y0, w, h = box_dict["box"]
-        x1 = x0 + w
-        y1 = y0 + h
-        verts = _get_bounding_box_plotting_vertices(x0, y0, x1, y1)
-        ax_im.plot(verts[:, 0] - crop_min_x, verts[:, 1], c=(0.0, 0.0, 1.0), alpha=0.3)
-        # c = max(float(box_dict["score"]) - 0.5, 0) * 2.0
-        # ax_im.plot(verts[:, 0] - crop_min_x, verts[:, 1], c=(1.0 - c, 1.0 - c, 1.0))
-        # ax_im.plot(verts[:, 0] - crop_min_x, verts[:, 1],
-        # c=(0.0, 0.0, 1.0), alpha=1.0)
-
-        # x1_large = x1 + 0.05 * w
-        # x0_large = x0 - 0.05 * w
-        # y1_large = y1 + 0.05 * w
-        # y0_large = y0 - 0.05 * w
-        # in_box_mask = np.logical_and(
-        #     np.logical_and(p_xy[0] > x0_large, p_xy[0] < x1_large),
-        #     np.logical_and(p_xy[1] > y0_large, p_xy[1] < y1_large)
-        # )
-        # neg_mask[in_box_mask] = False
-
-    for box in batch_dict["pseudo_label_boxes"][ib]:
-        x0, y0, x1, y1 = box
-        verts = _get_bounding_box_plotting_vertices(x0, y0, x1, y1)
-        ax_im.plot(verts[:, 0] - crop_min_x, verts[:, 1], c="green")
-
-    # overlay laser points on image
-    c_bgr = _distance_to_bgr_color(scan_r)
-    ax_im.scatter(
-        p_xy[0, ib_mask] - crop_min_x, p_xy[1, ib_mask], s=1, color=c_bgr[ib_mask]
-    )
-    # neg_mask = np.ones(p_xy.shape[1], dtype=np.bool)
-    # ax_im.scatter(p_xy[0, neg_mask] - crop_min_x, p_xy[1, neg_mask], s=1,
-    # color=c_bgr[neg_mask])
+    if show_pseudo_labels:
+        # detection bounding box
+        for box_dict in batch_dict["im_dets"][ib]:
+            x0, y0, w, h = box_dict["box"]
+            x1 = x0 + w
+            y1 = y0 + h
+            verts = _get_bounding_box_plotting_vertices(x0, y0, x1, y1)
+            ax_im.plot(
+                verts[:, 0] - crop_min_x, verts[:, 1], c=(0.0, 0.0, 1.0), alpha=0.3
+            )
+            # c = max(float(box_dict["score"]) - 0.5, 0) * 2.0
+            # ax_im.plot(verts[:, 0] - crop_min_x, verts[:, 1], c=(1.0 - c, 1.0 - c, 1.0))
+            # ax_im.plot(verts[:, 0] - crop_min_x, verts[:, 1],
+            # c=(0.0, 0.0, 1.0), alpha=1.0)
+
+            # x1_large = x1 + 0.05 * w
+            # x0_large = x0 - 0.05 * w
+            # y1_large = y1 + 0.05 * w
+            # y0_large = y0 - 0.05 * w
+            # in_box_mask = np.logical_and(
+            #     np.logical_and(p_xy[0] > x0_large, p_xy[0] < x1_large),
+            #     np.logical_and(p_xy[1] > y0_large, p_xy[1] < y1_large)
+            # )
+            # neg_mask[in_box_mask] = False
+
+        for box in batch_dict["pseudo_label_boxes"][ib]:
+            x0, y0, x1, y1 = box
+            verts = _get_bounding_box_plotting_vertices(x0, y0, x1, y1)
+            ax_im.plot(verts[:, 0] - crop_min_x, verts[:, 1], c="green")
+
+        # overlay only pseudo-label laser points on image
+        pl_pos_mask = np.logical_and(batch_dict["target_cls"][ib] == 1, ib_mask)
+        pl_neg_mask = np.logical_and(batch_dict["target_cls"][ib] == 0, ib_mask)
+        ax_im.scatter(
+            p_xy[0, pl_pos_mask] - crop_min_x, p_xy[1, pl_pos_mask], s=1, color="green",
+        )
+        ax_im.scatter(
+            p_xy[0, pl_neg_mask] - crop_min_x,
+            p_xy[1, pl_neg_mask],
+            s=1,
+            color="orange",
+        )
+
+        fig_file = os.path.join(_SAVE_DIR, f"figs/{sequence}/im_pl_{frame_id}.png")
+    else:
+        # overlay all laser points on image
+        c_bgr = _distance_to_bgr_color(scan_r)
+        ax_im.scatter(
+            p_xy[0, ib_mask] - crop_min_x, p_xy[1, ib_mask], s=1, color=c_bgr[ib_mask]
+        )
+        fig_file = os.path.join(_SAVE_DIR, f"figs/{sequence}/im_raw_{frame_id}.png")
 
     # save fig
-    fig_file = os.path.join(_SAVE_DIR, f"figs/{sequence}/im_{frame_id}.png")
     os.makedirs(os.path.dirname(fig_file), exist_ok=True)
     fig.savefig(fig_file, dpi=dpi)
     plt.close(fig)
 
 
-def _plot_frame_pts(batch_dict, ib, pred_cls, pred_reg):
+def _plot_frame_pts(batch_dict, ib, pred_cls, pred_reg, pred_cls_p, pred_reg_p):
     frame_id = f"{batch_dict['frame_id'][ib]:06d}"
     sequence = batch_dict["sequence"][ib]
 
@@ -159,18 +180,46 @@ def _plot_frame_pts(batch_dict, ib, pred_cls, pred_reg):
                 )
                 ax.add_artist(c)
 
-    # inference result or pseudo-labels
+    # plot detections
     if pred_cls is not None and pred_reg is not None:
         dets_xy, dets_cls, _ = u.nms_predicted_center(
             scan_r, scan_phi, pred_cls[ib].reshape(-1), pred_reg[ib]
         )
-        dets_xy = dets_xy[dets_cls > 0.15]
+        dets_xy = dets_xy[dets_cls >= 0.9438938]  # at EER
         if len(dets_xy) > 0:
             for x, y in dets_xy:
-                c = plt.Circle((x, y), radius=0.4, color=(0.0, 0.56, 0.56), fill=False)
+                c = plt.Circle((x, y), radius=0.4, color="green", fill=False)
                 ax.add_artist(c)
         fig_file = os.path.join(_SAVE_DIR, f"figs/{sequence}/scan_det_{frame_id}.png")
+
+        # plot in addition detections from a pre-trained
+        if pred_cls_p is not None and pred_reg_p is not None:
+            dets_xy, dets_cls, _ = u.nms_predicted_center(
+                scan_r, scan_phi, pred_cls_p[ib].reshape(-1), pred_reg_p[ib]
+            )
+            dets_xy = dets_xy[dets_cls > 0.29919282]  # at EER
+            if len(dets_xy) > 0:
+                for x, y in dets_xy:
+                    c = plt.Circle((x, y), radius=0.4, color="green", fill=False)
+                    ax.add_artist(c)
+    # plot pre-trained detections only
+    elif pred_cls_p is not None and pred_reg_p is not None:
+        dets_xy, dets_cls, _ = u.nms_predicted_center(
+            scan_r, scan_phi, pred_cls_p[ib].reshape(-1), pred_reg_p[ib]
+        )
+        dets_xy = dets_xy[dets_cls > 0.29919282]  # at EER
+        if len(dets_xy) > 0:
+            for x, y in dets_xy:
+                c = plt.Circle((x, y), radius=0.4, color="green", fill=False)
+                ax.add_artist(c)
+        fig_file = os.path.join(
+            _SAVE_DIR, f"figs/{sequence}/scan_pretrain_{frame_id}.png"
+        )
+    # plot pseudo-labels only
     else:
+        pl_neg_mask = batch_dict["target_cls"][ib] == 0
+        ax.scatter(scan_x[pl_neg_mask], scan_y[pl_neg_mask], s=0.5, c="orange")
+
         pl_xy = batch_dict["pseudo_label_loc_xy"][ib]
         if len(pl_xy) > 0:
             for x, y in pl_xy:
@@ -185,7 +234,7 @@ def _plot_frame_pts(batch_dict, ib, pred_cls, pred_reg):
 
 
 def plot_pseudo_label_for_all_frames():
-    with open("./base_drow_jrdb_cfg.yaml", "r") as f:
+    with open("./cfgs/base_drow_jrdb_cfg.yaml", "r") as f:
         cfg = yaml.safe_load(f)
     cfg["dataset"]["pseudo_label"] = True
     cfg["dataset"]["pl_correction_level"] = 0
@@ -203,10 +252,25 @@ def plot_pseudo_label_for_all_frames():
     model.eval()
 
     logger = Logger(cfg["pipeline"]["Logger"])
-    logger.load_ckpt("./ckpts/ckpt_phce_drow_e40.pth", model)
+    logger.load_ckpt("./ckpts/ckpt_jrdb_pl_drow3_phce_e40.pth", model)
+
+    model_pretrain = get_model(cfg["model"])
+    model_pretrain.cuda()
+    model_pretrain.eval()
+    logger.load_ckpt("./ckpts/ckpt_drow_drow3_e40.pth", model_pretrain)
 
     # generate pseudo labels for all sample
+    seq_count = 0
     for count, batch_dict in enumerate(tqdm(test_loader)):
+        if batch_dict["first_frame"][0]:
+            print(f"new seq, reset count, idx {count}")
+            seq_count = 0
+
+        if seq_count > _SEQ_MAX_COUNT:
+            continue
+        else:
+            seq_count += 1
+
         if count >= _MAX_COUNT:
             break
 
@@ -216,11 +280,25 @@ def plot_pseudo_label_for_all_frames():
             pred_cls = torch.sigmoid(pred_cls).data.cpu().numpy()
             pred_reg = pred_reg.data.cpu().numpy()
 
+            pred_cls_p, pred_reg_p = model_pretrain(net_input)
+            pred_cls_p = torch.sigmoid(pred_cls_p).data.cpu().numpy()
+            pred_reg_p = pred_reg_p.data.cpu().numpy()
+
         if count % _PLOTTING_INTERVAL == 0:
             for ib in range(len(batch_dict["input"])):
-                _plot_frame_im(batch_dict, ib)
-                _plot_frame_pts(batch_dict, ib, None, None)
-                _plot_frame_pts(batch_dict, ib, pred_cls, pred_reg)
+                # generate sequence videos
+                _plot_frame_im(batch_dict, ib, False)  # image
+                _plot_frame_im(batch_dict, ib, True)  # image
+                _plot_frame_pts(
+                    batch_dict, ib, None, None, None, None
+                )  # pseudo-label
+                _plot_frame_pts(
+                    batch_dict, ib, pred_cls, pred_reg, None, None
+                )  # detections
+
+                # # use to generate comparsion between pre-trained and pseudo-label trained  # noqa
+                # _plot_frame_pts(batch_dict, ib, pred_cls, pred_reg, None, None)
+                # _plot_frame_pts(batch_dict, ib, None, None, pred_cls_p, pred_reg_p)
 
 
 def plot_color_bar():

dr_spaam/bin/plotting/make_videos.sh

@@ -0,0 +1,16 @@
+for DIR in figs/*; do
+    if [[ ! -d $DIR ]]; then
+        continue
+    fi
+
+    if [[ ! -d videos ]]; then
+        mkdir videos
+    fi
+
+    seq=$(basename -- $DIR)
+
+    ffmpeg -r 20 -pattern_type glob -i "${DIR}/im_raw_*.png" -c:v libx264 -vf fps=30 -pix_fmt yuv420p "videos/${seq}__im_raw.mp4"
+    ffmpeg -r 20 -pattern_type glob -i "${DIR}/im_pl_*.png" -c:v libx264 -vf fps=30 -pix_fmt yuv420p "videos/${seq}__im_pl.mp4"
+    ffmpeg -r 20 -pattern_type glob -i "${DIR}/scan_det_*.png" -c:v libx264 -vf fps=30 -pix_fmt yuv420p "videos/${seq}__det.mp4"
+    ffmpeg -r 20 -pattern_type glob -i "${DIR}/scan_pl_*.png" -c:v libx264 -vf fps=30 -pix_fmt yuv420p "videos/${seq}__pl.mp4"
+done