add new files

get_projection_maps.py

@@ -0,0 +1,65 @@
+"""
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Manually select points to get the projection map
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+"""
+import argparse
+import os
+import numpy as np
+import cv2
+from surround_view import FisheyeCameraModel, PointSelector, display_image
+import surround_view.param_settings as settings
+
+
+def get_projection_map(camera_model, image, dst_points):
+    image = camera_model.undistort(image)
+    gui = PointSelector(image, title=camera_model.camera_name)
+    choice = gui.loop()
+    if choice > 0:
+        src = np.float32(gui.keypoints)
+        dst = np.float32(dst_points)
+        camera_model.project_matrix = cv2.getPerspectiveTransform(src, dst)
+        image = camera_model.project(image)
+
+        ret = display_image("Bird's View", image)
+        if ret > 0:
+            return True
+        if ret < 0:
+            cv2.destroyAllWindows()
+
+    return False
+
+
+parser = argparse.ArgumentParser()
+parser.add_argument("-camera", required=True,
+                    choices=["front", "back", "left", "right"],
+                    help="The camera view to be projected")
+parser.add_argument("-scale", nargs="+", default=None,
+                    help="scale the undistorted image")
+parser.add_argument("-shift", nargs="+", default=None,
+                    help="shift the undistorted image")
+args = parser.parse_args()
+
+if args.scale is not None:
+    scale = [float(x) for x in args.scale]
+else:
+    scale = (1.0, 1.0)
+
+if args.shift is not None:
+    shift = [float(x) for x in args.shift]
+else:
+    shift = (0, 0)
+
+camera_name = args.camera
+camera_file = os.path.join(os.getcwd(), "yaml", camera_name + ".yaml")
+image_file = os.path.join(os.getcwd(), "images", camera_name + ".png")
+image = cv2.imread(image_file)
+dst_points = settings.dst_points[camera_name]
+camera = FisheyeCameraModel(camera_file, camera_name)
+camera.set_scale_and_shift(scale, shift)
+success = get_projection_map(camera, image, dst_points)
+if success:
+    print("saving projection matrix to yaml")
+
+else:
+    print("failed to compute the projection map")

get_weight_matrices.py

@@ -0,0 +1,31 @@
+import os
+import numpy as np
+import cv2
+from PIL import Image
+from surround_view import FisheyeCameraModel, display_image, BirdView
+
+
+names = ["front", "back", "left", "right"]
+images = [os.path.join(os.getcwd(), "images", name + ".png") for name in names]
+yamls = [os.path.join(os.getcwd(), "yaml", name + ".yaml") for name in names]
+camera_models = [FisheyeCameraModel(camera_file, camera_name)
+                 for camera_file, camera_name in zip (yamls, names)]
+
+projected = []
+for image_file, camera in zip(images, camera_models):
+    img = cv2.imread(image_file)
+    img = camera.undistort(img)
+    img = camera.project(img)
+    img = camera.flip(img)
+    projected.append(img)
+
+birdview = BirdView()
+Gmat, Mmat = birdview.get_weights(projected)
+projected2 = birdview.make_luminance_balance(projected)
+birdview.stitch_all_parts(projected2)
+birdview.make_white_balance()
+birdview.copy_car_image()
+ret = display_image("BirdView Result", birdview.image)
+if ret > 0:
+    Image.fromarray((Gmat * 255).astype(np.uint8)).save("weights.png")
+    Image.fromarray(Mmat.astype(np.uint8)).save("masks.png")

surround_view/__init__.py

@@ -0,0 +1,3 @@
+from .fisheye_camera import FisheyeCameraModel
+from .birdview import BirdView
+from .gui import display_image, PointSelector

surround_view/birdview.py

@@ -0,0 +1,217 @@
+import numpy as np
+import cv2
+import os
+from PyQt5.QtCore import QMutex
+from .base_thread import BaseThread
+from .buffer import Buffer
+from .param_settings import *
+from .utils import *
+
+
+def FI(front_image):
+    return front_image[:, :xl]
+
+
+def FII(front_image):
+    return front_image[:, xr:]
+
+
+def FM(front_image):
+    return front_image[:, xl:xr]
+
+
+def BIII(back_image):
+    return back_image[:, :xl]
+
+
+def BIV(back_image):
+    return back_image[:, xr:]
+
+
+def BM(back_image):
+    return back_image[:, xl:xr]
+
+
+def LI(left_image):
+    return left_image[:yt, :]
+
+
+def LIII(left_image):
+    return left_image[yb:, :]
+
+
+def LM(left_image):
+    return left_image[yt:yb, :]
+
+
+def RII(right_image):
+    return right_image[:yt, :]
+
+
+def RIV(right_image):
+    return right_image[yb:, :]
+
+
+def RM(right_image):
+    return right_image[yt:yb, :]
+
+
+class BirdView(BaseThread):
+
+    def __init__(self,
+                 receive_buffer_manager=None,
+                 drop_if_full=True,
+                 buffer_size=8,
+                 parent=None):
+        super(BirdView, self).__init__(parent)
+        self.image = np.zeros((total_h, total_w, 3), np.uint8)
+        self.weights = None
+        self.receive_buffer_manager = receive_buffer_manager
+        self.drop_if_full = drop_if_full
+        self.buffer = Buffer(buffer_size)
+        self.car_image = cv2.imread(os.path.join(os.getcwd(), "images", "car.png"))
+        self.car_image = cv2.resize(self.car_image, (xr - xl, yb - yt))
+
+    def merge(self, imA, imB, k):
+        G = self.weights[k]
+        return (imA * G + imB * (1 - G)).astype(np.uint8)
+
+    @property
+    def FL(self):
+        return self.image[:yt, :xl]
+
+    @property
+    def F(self):
+        return self.image[:yt, xl:xr]
+
+    @property
+    def FR(self):
+        return self.image[:yt, xr:]
+
+    @property
+    def BL(self):
+        return self.image[yb:, :xl]
+
+    @property
+    def B(self):
+        return self.image[yb:, xl:xr]
+
+    @property
+    def BR(self):
+        return self.image[yb:, xr:]
+
+    @property
+    def L(self):
+        return self.image[yt:yb, :xl]
+
+    @property
+    def R(self):
+        return self.image[yt:yb, xr:]
+
+    @property
+    def C(self):
+        return self.image[yt:yb, xl:xr]
+
+    def stitch_all_parts(self, images):
+        front, back, left, right = images
+        np.copyto(self.F, FM(front))
+        np.copyto(self.B, BM(back))
+        np.copyto(self.L, LM(left))
+        np.copyto(self.R, RM(right))
+        np.copyto(self.FL, self.merge(FI(front), LI(left), 0))
+        np.copyto(self.FR, self.merge(FII(front), RII(right), 1))
+        np.copyto(self.BL, self.merge(BIII(back), LIII(left), 2))
+        np.copyto(self.BR, self.merge(BIV(back), RIV(right), 3))
+
+    def load_weights(self, weights_image_path, masks_image_path):
+        from PIL import Image
+
+        GMat = np.asarray(Image.open(weights_image_path).convert("RGBA"), dtype=np.float) / 255.0
+        self.weights = [np.stack((GMat[:, :, k],
+                                  GMat[:, :, k],
+                                  GMat[:, :, k]), axis=2)
+                        for k in range(4)]
+
+        Mmat = np.asarray(Image.open(masks_image_path).convert("RGBA"), dtype=np.float) / 255.0
+        self.masks = [Mmat.astype(np.int)[:, :, k] for k in range(4)]
+
+    def run(self):
+        if self.receive_buffer_manager is None:
+            raise ValueError("This thread requires a buffer of projected images to run")
+
+        while True:
+            self.stop_mutex.lock()
+            if self.stopped:
+                self.stopped = False
+                self.stop_mutex.unlock()
+                break
+            self.stop_mutex.unlock()
+            self.processing_time = self.clock.elapsed()
+            self.clock.start()
+
+            self.processing_mutex.lock()
+
+            frames = self.receive_buffer_manager.get()
+            frame_images = frames.values()
+            frame_images = self.make_luminance_balance(frame_images)
+            self.stitch_all_parts(frame_images)
+            self.make_white_balance()
+            self.copy_car_image()
+            self.buffer.add(self.image.copy(), self.drop_if_full)
+            self.processing_mutex.unlock()
+
+            # update statistics
+            self.update_fps(self.processing_time)
+            self.stat_data.frames_processed_count += 1
+            # inform GUI of updated statistics
+            self.update_statistics_gui.emit(self.stat_data)
+
+    def copy_car_image(self):
+        np.copyto(self.C, self.car_image)
+
+    def get_weights(self, images):
+        front, back, left, right = images
+        G0, M0 = get_weight_matrix(FI(front), LI(left))
+        G1, M1 = get_weight_matrix(FII(front), RII(right))
+        G2, M2 = get_weight_matrix(BIII(back), LIII(left))
+        G3, M3 = get_weight_matrix(BIV(back), RIV(right))
+        self.weights = [np.stack((G, G, G), axis=2) for G in (G0, G1, G2, G3)]
+        self.masks = [(M / 255.0).astype(np.int) for M in (M0, M1, M2, M3)]
+        return np.stack((G0, G1, G2, G3), axis=2), np.stack((M0, M1, M2, M3), axis=2)
+
+    def make_white_balance(self):
+        self.image = make_white_balance(self.image)
+
+    def make_luminance_balance(self, images):
+        front, back, left, right = images
+        M0, M1, M2, M3 = self.masks
+        a1, a2, a3 = rgb_ratio(RII(right), FII(front), M1)
+        b1, b2, b3 = rgb_ratio(BIV(back), RIV(right), M3)
+        c1, c2, c3 = rgb_ratio(LIII(left), BIII(back), M2)
+        d1, d2, d3 = rgb_ratio(FI(front), LI(left), M0)
+
+        t1 = (a1*b1*c1*d1)**0.25
+        t2 = (a2*b2*c2*d2)**0.25
+        t3 = (a3*b3*c3*d3)**0.25
+
+        e1 = t1 / (d1/a1)**0.5
+        e2 = t2 / (d2/a2)**0.5
+        e3 = t3 / (d3/a3)**0.5
+        front = adjust_luminance_rgb(front, (e1, e2, e3))
+
+        f1 = t1 / (b1/c1)**0.5
+        f2 = t2 / (b2/c2)**0.5
+        f3 = t3 / (b3/c3)**0.5
+        back = adjust_luminance_rgb(back, (f1, f2, f3))
+
+        g1 = t1 / (c1/d1)**0.5
+        g2 = t2 / (c2/d2)**0.5
+        g3 = t3 / (c3/d3)**0.5
+        left = adjust_luminance_rgb(left, (g1, g2, g3))
+
+        h1 = t1 / (a1/b1)**0.5
+        h2 = t2 / (a2/b2)**0.5
+        h3 = t3 / (a3/b3)**0.5
+        right = adjust_luminance_rgb(right, (h1, h2, h3))
+
+        return [front, back, left, right]

surround_view/gui.py

@@ -86,7 +86,7 @@ def loop(self):
         cv2.imshow(self.title, self.image)
 
         while True:
-            click = cv2.getWindowProperty(window_title, cv2.WND_PROP_AUTOSIZE)
+            click = cv2.getWindowProperty(self.title, cv2.WND_PROP_AUTOSIZE)
             if click < 0:
                 return False
 

surround_view/param_settings.py

@@ -0,0 +1,65 @@
+# (shift_width, shift_height): how far away the birdview looks outside
+# of the calibration pattern in horizontal and vertical directions
+shift_w = 300
+shift_h = 300
+
+# size of the gap between the calibration pattern and the car
+# in horizontal and vertical directions
+inn_shift_w = 20
+inn_shift_h = 50
+
+# total width/height of the stitched image
+total_w = 600 + 2 * shift_w
+total_h = 1000 + 2 * shift_h
+
+# four corners of the rectangular region occupied by the car
+# top-left (x_left, y_top), bottom-right (x_right, y_bottom)
+xl = shift_w + 180 + inn_shift_w
+xr = total_w - xl
+yt = shift_h + 200 + inn_shift_h
+yb = total_h - yt
+
+# ------------------------------------------------------------
+camera_names = ("front", "back", "left", "right")
+
+front_shape = (total_w, yt)
+back_shape = front_shape
+left_shape = (total_h, xl)
+right_shape = left_shape
+
+project_shapes = {
+    "front": front_shape,
+    "back": back_shape,
+    "left": left_shape,
+    "right": right_shape
+}
+
+# pixel locations of the four points to be choosen.
+# you must click these pixels in the same order when running
+# the get_projection_map.py script.
+front_proj_keypoints = [(shift_w + 120, shift_h),
+                        (shift_w + 480, shift_h),
+                        (shift_w + 120, shift_h + 160),
+                        (shift_w + 480, shift_h + 160)]
+
+back_proj_keypoints = [(shift_w + 120, shift_h),
+                       (shift_w + 480, shift_h),
+                       (shift_w + 120, shift_h + 160),
+                       (shift_w + 480, shift_h + 160)]
+
+left_proj_keypoints = [(shift_h + 280, shift_w),
+                       (shift_h + 840, shift_w),
+                       (shift_h + 280, shift_w + 160),
+                       (shift_h + 840, shift_w + 160)]
+
+right_proj_keypoints = [(shift_h + 160, shift_w),
+                        (shift_h + 720, shift_w),
+                        (shift_h + 160, shift_w + 160),
+                        (shift_h + 720, shift_w + 160)]
+
+dst_points = {
+    "front": front_proj_keypoints,
+    "back": back_proj_keypoints,
+    "left": left_proj_keypoints,
+    "right": right_proj_keypoints
+}