Add dolly-zoom effect.

README.md

@@ -65,7 +65,10 @@ For more detailed instructions, please refer to [`DOCUMENTATION.md`](DOCUMENTATI
     - Rendered videos with swing motion
         - E.g. ```video/moon_swing.mp4```
     - Rendered videos with circle motion
-        - E.g. ```video/moon_circle.mp4```              
+        - E.g. ```video/moon_circle.mp4```         
+    - Rendered videos with dolly zoom-in effect
+        - E.g. ```video/moon_dolly-zoom-in.mp4```
+        - Note: We choose the depth value locate at image center.
 4. (Optional) If you want to change the default configuration. Please read [`DOCUMENTATION.md`](DOCUMENTATION.md) and modified ```argument.yml```.
 
 

argument.yml

@@ -4,11 +4,11 @@ rgb_feat_model_ckpt: checkpoints/color-model.pth
 MiDaS_model_ckpt: MiDaS/model.pt
 fps: 40
 num_frames: 240
-x_shift_range: [-0.03, -0.02, -0.02]
-y_shift_range: [-0.00, -0.00, -0.02]
-z_shift_range: [-0.07, -0.07, -0.07]
-traj_types: ['straight-line', 'circle', 'circle']
-video_postfix: ['zoom-in', 'swing', 'circle']
+x_shift_range: [0.00, 0.00, -0.02, -0.02]
+y_shift_range: [0.00, 0.00, -0.02, -0.00]
+z_shift_range: [-0.05, -0.05, -0.07, -0.07]
+traj_types: ['double-straight-line', 'double-straight-line', 'circle', 'circle']
+video_postfix: ['dolly-zoom-in', 'zoom-in', 'circle', 'swing']
 specific: ''
 longer_side_len: 960
 src_folder: image

main.py

@@ -55,9 +55,10 @@
         config['gray_image'] = True
     else:
         config['gray_image'] = False
+    image = cv2.resize(image, (config['output_w'], config['output_h']), interpolation=cv2.INTER_AREA)
+    depth = read_MiDaS_depth(sample['depth_fi'], 3.0, config['output_h'], config['output_w'])
+    mean_loc_depth = depth[depth.shape[0]//2, depth.shape[1]//2]
     if not(config['load_ply'] is True and os.path.exists(mesh_fi)):
-        image = cv2.resize(image, (config['output_w'], config['output_h']), interpolation=cv2.INTER_AREA)
-        depth = read_MiDaS_depth(sample['depth_fi'], 3.0, config['output_h'], config['output_w'])
         vis_photos, vis_depths = sparse_bilateral_filtering(depth.copy(), image.copy(), config, num_iter=config['sparse_iter'], spdb=False)
         depth = vis_depths[-1]
         model = None
@@ -112,4 +113,5 @@
     normal_canvas, all_canvas = output_3d_photo(verts.copy(), colors.copy(), faces.copy(), copy.deepcopy(Height), copy.deepcopy(Width), copy.deepcopy(hFov), copy.deepcopy(vFov),
                         copy.deepcopy(sample['tgt_pose']), sample['video_postfix'], copy.deepcopy(sample['ref_pose']), copy.deepcopy(config['video_folder']),
                         image.copy(), copy.deepcopy(sample['int_mtx']), config, image,
-                        videos_poses, video_basename, config.get('original_h'), config.get('original_w'), border=border, depth=depth, normal_canvas=normal_canvas, all_canvas=all_canvas)
+                        videos_poses, video_basename, config.get('original_h'), config.get('original_w'), border=border, depth=depth, normal_canvas=normal_canvas, all_canvas=all_canvas,
+                        mean_loc_depth=mean_loc_depth)

mesh.py

@@ -895,17 +895,6 @@ def context_and_holes(mesh, edge_ccs, config, specific_edge_id, specific_edge_lo
     for edge_id, edge_cc in enumerate(edge_ccs):
         for edge_node in edge_cc:
             edge_maps[edge_node[0], edge_node[1]] = edge_id
-    mask_time = 0
-    context_time = 0
-    intouch_time = 0
-    redundant_time = 0
-    noncont_time = 0
-    ext_first_time = 0
-    ext_modified_time = 0
-    ext_mask_time = 0
-    ext_context_time = 0
-    ext_copy_time = 0
-    inpaint_depth_time = 0
 
     context_ccs = [set() for x in range(len(edge_ccs))]
     extend_context_ccs = [set() for x in range(len(edge_ccs))]
@@ -1340,9 +1329,7 @@ def context_and_holes(mesh, edge_ccs, config, specific_edge_id, specific_edge_lo
                             bool(tmp_mask_map[ne[0], ne[1]]) is False and \
                             bool(tmp_context_map[ne[0], ne[1]]) is False and \
                             bool(forbidden_map[ne[0], ne[1]]) is True:
-
                             new_tmp_context_nodes.append(ne)
-
                             tmp_context_map[ne[0], ne[1]] = True
                 accum_context_cc.extend(new_tmp_context_nodes)
                 for node in tmp_invalid_context_nodes:
@@ -2164,7 +2151,7 @@ def reinit_camera(self, fov):
 
 def output_3d_photo(verts, colors, faces, Height, Width, hFov, vFov, tgt_poses, video_traj_types, ref_pose,
                     output_dir, ref_image, int_mtx, config, image, videos_poses, video_basename, original_H=None, original_W=None,
-                    border=None, depth=None, normal_canvas=None, all_canvas=None):
+                    border=None, depth=None, normal_canvas=None, all_canvas=None, mean_loc_depth=None):
 
     cam_mesh = netx.Graph()
     cam_mesh.graph['H'] = Height
@@ -2230,6 +2217,7 @@ def output_3d_photo(verts, colors, faces, Height, Width, hFov, vFov, tgt_poses,
                   0,
                   img.shape[1]]
     anchor = np.array(anchor) * factor
+    plane_width = np.tan(fov_in_rad/2.) * np.abs(mean_loc_depth)
     for video_pose, video_traj_type in zip(videos_poses, video_traj_types):
         stereos = []
         tops = []; buttoms = []; lefts = []; rights = []
@@ -2238,6 +2226,12 @@ def output_3d_photo(verts, colors, faces, Height, Width, hFov, vFov, tgt_poses,
             axis, angle = transforms3d.axangles.mat2axangle(rel_pose[0:3, 0:3])
             normal_canvas.rotate(axis=axis, angle=(angle*180)/np.pi)
             normal_canvas.translate(rel_pose[:3,3])
+            new_mean_loc_depth = mean_loc_depth - float(rel_pose[2, 3])
+            if 'dolly' in video_traj_type:
+                new_fov = float((np.arctan2(plane_width, np.array([np.abs(new_mean_loc_depth)])) * 180. / np.pi) * 2)
+                normal_canvas.reinit_camera(new_fov)
+            else:
+                normal_canvas.reinit_camera(fov)
             normal_canvas.view_changed()
             img = normal_canvas.render()
             img = cv2.resize(img, (int(img.shape[1] / init_factor), int(img.shape[0] / init_factor)), interpolation=cv2.INTER_CUBIC)

utils.py

@@ -31,6 +31,13 @@ def path_planning(num_frames, x, y, z, path_type=''):
         cs = interp1d(corner_t, corner_points, axis=0, kind='quadratic')
         spline = cs(t)
         xs, ys, zs = [xx.squeeze() for xx in np.split(spline, 3, 1)]
+    elif path_type == 'double-straight-line':
+        corner_points = np.array([[-x, -y, -z], [0, 0, 0], [x, y, z]])
+        corner_t = np.linspace(0, 1, len(corner_points))
+        t = np.linspace(0, 1, num_frames)
+        cs = interp1d(corner_t, corner_points, axis=0, kind='quadratic')
+        spline = cs(t)
+        xs, ys, zs = [xx.squeeze() for xx in np.split(spline, 3, 1)]        
     elif path_type == 'circle':
         xs, ys, zs = [], [], []
         for frame_id, bs_shift_val in enumerate(np.arange(-2.0, 2.0, (4./num_frames))):