update image matching module

README.md

@@ -33,7 +33,7 @@ This submission consists of various methods for video stitching from multi-camer
 
 - `feature_matching_test.py` - Apply Brute-Force Matching and KNN Matching methods for all features from two images, then apply RANSAC to estimate the Inlier Matches
 
-- `ROIs_matching_test.py` - Apply Brute-Force Matching and KNN Matching methods for features within the target corresponding Range of Interests(ROIs). The filtered features in one region can only match to the specific region in another image. Match multiple corresponding areas separately for two images, then apply RANSAC to estimate the Inlier Matches
+- `ROIs_matching_test.py` - Apply Brute-Force Matching and KNN Matching methods for features between corresponding Range of Interests(ROIs). The filtered features in one region can only match to the specific region in another image. Match multiple corresponding areas separately for two images, then apply RANSAC to estimate the Inlier Matches
 
 - `undistortion_stitch_test.py` - The features generated from the calibrated images may be unreal due to image stretching or compression in undistortion process. This script makes use of the initial features from original fisheye images to estimate the homography matrix, then perform the image stitching on undistortion images
 
@@ -54,7 +54,7 @@ This submission consists of various methods for video stitching from multi-camer
     $ python feature_extraction_test.py
 ```
 <div align="center">
-<img src="result/feature_extraction.png" width="450" height="275"/>
+<img src="result/feature_extraction.png" width="450" height="250"/>
 <br/>
 Figure 1  Feature Extraction Comparison
 </div>
@@ -65,15 +65,20 @@ Figure 1  Feature Extraction Comparison
     $ python feature_matching_test.py
 ```
 <div align="center">
-<img src="result/feature_matching.png" width="450" height="250"/>
+<img src="result/feature_matching.png" width="450" height="225"/>
 <br/>
-Figure 2  Inlier Matches with BF/KNN Matching Methods 
+Figure 2  Inlier Matches with BF/KNN Matching Methods for All Features 
 </div>
 
-- ROIs Matching Test: Match the features within corresponding areas to avoid wrong matches across different regions. The number of inliers has slightly increased as a result of the Separate Region Processing
+- ROIs Matching Test: Match the features within corresponding areas to avoid wrong matches across different regions. As a result of the Separate Region Processing, false matches and computation time will be effectively reduced, and correct matching inliers will be slightly increased.
 ```bash
     $ python ROIs_matching_test.py
 ```
+<div align="center">
+<img src="result/ROIs.png" width="450" height="200"/>
+<br/>
+Figure 3  Inlier Matches with corresponding ROI Mathcing for Selected Features
+</div>
 
 ### Image Stitching
 - Undistortion Stitch Test: Stitch Images using original features to estimate homography matrix
@@ -83,10 +88,10 @@ Figure 2  Inlier Matches with BF/KNN Matching Methods
 <div align="center">
 <img src="result/distorted_ROI.png" width="400" height="200"/>
 <br/>
-Figure 3  Increased Inlier Matches by ROIs corresponding Matching on Original Images
+Figure 4  Increased Inlier Matches by ROIs corresponding Matching on Original Images
 </div>
 
-> This module requires original fisheye images and camera calibration parameter for undistortion process.
+> This module requires original fisheye images and camera calibration parameters for undistortion process.
 
 ### Video Stitching
 

feature_matching_test.py

@@ -26,32 +26,12 @@
 
     # Initialize the Stitch Class
     Img1 = Image(img1)
-    Img2 = Image(img2)
-
-    # Manually define the ROI to locate the area for corresponding images
-    cv.namedWindow("Area Selection", cv.WINDOW_NORMAL)
-    cv.resizeWindow("Area Selection", 800, 600)
-    ROIs1 = cv.selectROIs("Area Selection", img1)
-    ROIs2 = cv.selectROIs("Area Selection", img2)
-
-    for i in range(len(ROIs1)):
-        ROIs1[i, 2] = ROIs1[i, 0] + ROIs1[i, 2]
-        ROIs1[i, 3] = ROIs1[i, 1] + ROIs1[i, 3]
-        ROIs2[i, 2] = ROIs2[i, 0] + ROIs2[i, 2]
-        ROIs2[i, 3] = ROIs2[i, 1] + ROIs2[i, 3]
-
-
+    Img2 = Image(img2)       
+
     '''SIFT Features Matching Comparison'''
     # Find SIFT features of matching images
-    kps1_sift, dps1_sift = Img1.kps, Img1.des
-    kps2_sift, dps2_sift = Img2.kps, Img2.des
-
-    # Filter the Features
-    masks1 = utils.getMaskPointsInROIs(kps1_sift, ROIs1)
-    masks2 = utils.getMaskPointsInROIs(kps2_sift, ROIs2)
-
-    kps1_filter, des1_filter = Img1.featureFilter(masks1[0])
-    kps2_filter, des2_filter = Img2.featureFilter(masks2[0])
+    kps1_filter, des1_filter = Img1.kps, Img1.des
+    kps2_filter, des2_filter = Img2.kps, Img2.des
 
 
     # BFMatches(des1_filter, des2_filter)
@@ -63,18 +43,10 @@
     img_sift_knn = cv.drawMatches(Img1.img,kps1_filter,Img2.img,kps2_filter,matches_sift_knn[:300],None,**draw_params)
 
 
-
     '''BRISK Features Matching Comparison'''
     # Extract the BRISK features
-    kps1_brisk, dps1_brisk = Img1.findFeatures('brisk')
-    kps2_brisk, dps2_brisk = Img2.findFeatures('brisk')
-
-    # Filter the Features
-    mask1_brisk = utils.getMaskPointsInROIs(kps1_brisk,ROIs1)
-    kps1_filter_, des1_filter_ = Img1.featureFilter(mask1_brisk[0])
-
-    mask2_brisk = utils.getMaskPointsInROIs(kps2_brisk,ROIs2)
-    kps2_filter_, des2_filter_ = Img2.featureFilter(mask2_brisk[0])
+    kps1_filter_, des1_filter_ = Img1.findFeatures('brisk')
+    kps2_filter_, des2_filter_ = Img2.findFeatures('brisk')
 
     # BFMatches(des1_filter, des2_filter)
     matches_brisk = utils.featureMatch(des1_filter_, des2_filter_, 'brisk')

stitch/ImageStitch.py

@@ -150,7 +150,23 @@ def featureIntegrate(self, matches_list):
         self.Img1.kps = kps1
         self.Img2.kps = kps2
         self.matches = matchInt
+
+def remapStitch(img1, img2, u,v,map_x, map_y,mask):
+    # x_range = np.arange(0, stitch_size[0]+1)
+    # y_range = np.arange(0, stitch_size[1]+1)
+    # u, v = np.meshgrid(x_range, y_range)
+    # u = np.float32(u)
+    # v = np.float32(v)
+    img_super = cv.remap(img1, u, v, cv.INTER_LINEAR)
+    img_transform = cv.remap(img2, map_x, map_y, cv.INTER_LINEAR)
 
+    #img_transform[high_y:high_y + 200, :, :] = 0
+    #img_super[mask] = 0
+    img_transform[:600, :, :] = 0
+    img_super[600:, :, :] = 0
+
+    img_stitch = img_transform + img_super
+    return img_stitch
 
 def simpleStitch(img1, img2, homo_mat):
     # Get the position of vertices
@@ -169,9 +185,9 @@ def simpleStitch(img1, img2, homo_mat):
 
     homo_mat_ = np.eye(3)
     img_super = cv.warpPerspective(
-        img1, homo_mat_, stitch_size, borderValue=(0, 0, 0))
+        img1, homo_mat_, stitch_size)
     img_transform = cv.warpPerspective(
-        img2, homo_mat, stitch_size, borderValue=(0, 0, 0))
+        img2, homo_mat, stitch_size)
 
 
     # Combine the image on one super image

stitch/__init__.py

@@ -1,9 +1,9 @@
 from .utils import getParams
-from .ImageStitch import Image, Stitch, simpleStitch, alphablend
+from .ImageStitch import Image, Stitch, simpleStitch, alphablend, remapStitch
 from .PositioningSystem import getPos, getPos_box, getPos_box_array
 from .undistortion import load_params, calculate_map, undistort, feature_map
 
 __all__ = ['getParams',
-           'Image','Stitch','simpleStitch','alphablend',
+           'Image','Stitch','simpleStitch','alphablend','remapStitch',
            'getPos','getPos_box','getPos_box_array',
            'load_params','calculate_map','undistort','feature_map']

stitch_custom.py

@@ -11,6 +11,7 @@
 import argparse
 import stitch.ImageStitch as ImageStitch
 import numpy as np
+import time
 from stitch import getParams
 from pathlib import Path
 from vid_sync import VideoSynchronizer
@@ -58,7 +59,7 @@ def stitchImages(imgs, homo_params, stitch_params, farm_name):
     
     :return  img_stitch: stitching image of two images
     '''
-
+    T1 = time.time()
     # Initialize the size of the panorama image (for visualization)
     panorama_size = (stitch_params["panorama_size"][0], stitch_params["panorama_size"][1])
 
@@ -124,7 +125,8 @@ def stitchImages(imgs, homo_params, stitch_params, farm_name):
                 panorama_pos = param["value"][0]
                 stitch_pos = param["value"][1]
                 panorama[panorama_pos[0]:panorama_pos[1], panorama_pos[2]:panorama_pos[3],:] = img[stitch_pos[0]:stitch_pos[1], stitch_pos[2]:stitch_pos[3],:]
-
+    T2 = time.time()
+    print("Stitch Images Time:",(T2-T1)*1000)
     return panorama
 
 

warp_testing.py

@@ -0,0 +1,65 @@
+import time
+import cv2 as cv
+import numpy as np
+from stitch import remapStitch, simpleStitch
+from stitch.utils import transformVerts
+
+img1 = cv.imread("dataset/Arie/lamp_02_Arie.PNG")
+img2 = cv.imread("dataset/Arie/lamp_01_Arie.PNG")
+
+homo_mat = np.array([
+        1.1244483061556527, 0.020214811271289677, 39.9989392156668,
+        0.07199851056999242, 1.0588530742318294, 403.15979303614347,
+        0.00010189396429253117, 0.00003398322398670004, 1.0]).reshape(3,3)
+
+img_size = np.array([img2.shape[1],img2.shape[0]])
+vertice = transformVerts(img_size, homo_mat)
+x_min = vertice[:, 0].min()
+x_max = vertice[:, 0].max()
+y_min = vertice[:, 1].min()
+y_max = vertice[:, 1].max()
+# print("x_min: %d, x_max: %d y_min: %d, y_max: %d" %
+#       (x_min, x_max, y_min, y_max))
+
+# Define the size of the result image
+x_max = np.max([x_max, img2.shape[1]])
+y_max = np.max([y_max, img2.shape[0]])
+stitch_size = (x_max, y_max)
+
+img_transform = cv.warpPerspective(
+        img2, homo_mat, stitch_size, borderValue=(0, 0, 0))
+mask = img_transform>0
+print(mask.shape)
+print(mask)
+
+T1 = time.time()
+for i in range(100):
+    img_stitch = simpleStitch(img1, img2, homo_mat,mask)
+T2 = time.time()
+cv.imshow("stitch",img_stitch)
+
+x_range = np.arange(0, x_max)
+y_range = np.arange(0, y_max)
+u, v = np.meshgrid(x_range, y_range)
+u = np.float32(u)
+v = np.float32(v)
+
+homo_mat = np.linalg.inv(homo_mat)
+# warped_img = cv.remap(img1, map_x, map_y, cv.INTER_LINEAR, borderMode=cv.BORDER_REFLECT_101)
+# mask1 = np.ones((img1.shape[0],img1.shape[1]))
+# warped_img = cv.remap(mask1, map_x, map_y, cv.INTER_LINEAR, borderMode=cv.BORDER_REFLECT_101)
+z_ = homo_mat[2,0]*u + homo_mat[2,1]*v + homo_mat[2,2]
+map_x = (homo_mat[0,0]*u + homo_mat[0,1]*v + homo_mat[0,2])/z_
+map_y = (homo_mat[1,0]*u + homo_mat[1,1]*v + homo_mat[1,2])/z_
+
+T3 = time.time()
+for i in range(100):
+    img_stitch = remapStitch(img1,img2,u,v,map_x,map_y,mask)
+T4 = time.time()
+cv.imshow("warped",img_stitch)
+cv.waitKey(0)
+print("simpleStitch:",(T2-T1)*10)
+print("remapStitch:",(T4-T3)*10)
+
+
+