update feature_matching and feature_extraction test scripts

README.md

@@ -11,14 +11,14 @@ This submission consists of various methods for video stitching from multi-camer
 ## Files Description
     .
     ├── Result                        # Folder for Animation and Image Demonstrations
-    │ 
     ├── stitch
     |      ├─── __init__.py
     |      ├─── ImageStitch.py        # Define the Image and Stitch class
     |      ├─── PositioningSystem.py  # Transformation Function from Local Image to Global Image
     |      └─── utils.py              # Basic functions for stitching
     |
     ├── feature_extraction_test.py    # Image Preprocessing and Feature Extraction
+    ├── feature_matching_test.py      # Feature Matching and Inliers Estimation
     ├── panorama_test.py              # Generate panorama image for one frame
     ├── PositioningSystem_Test.py     # Test Script for Visualizing the Positioning System on Hard-Coded Points
     ├── stitch_custom.py              # Script for Real-time Video Stitching using generalized stitching function with stitching params input
@@ -27,19 +27,28 @@ This submission consists of various methods for video stitching from multi-camer
     ├── LICENSE
     └── README.md
 
-- `feature_extraction_test.py` - Apply Histogram Equalization for image preprocessing, then extract the SIFT, BRISK and orb features from image for comparison
+- `feature_extraction_test.py` - Apply Histogram Equalization for image preprocessing, then extract the SIFT, BRISK and ORB features from image for comparison
+- `feature_matching_test.py` - Apply Brute-Force Matching and KNN Matching methods for feature matching from two images, then apply RANSAC to estimate the Inlier Matches
 - `panorama_test.py` - Stitch the new image with input stitching combination, using Perspective Transform to stitch the left whole area and the right whole area
 - `ImageStitch.py` - Define the `Image` class which combines properties and functions for feature processing on one image, and `Stitch` class which combines properties and functions for matches and features on a pair of images
 - `stitch_custom.py` - Given the undistortion videos of multiple cameras, utilize the estimated homography parameters generated from `panorama_test.py` to stitch the image of every frame to create a panorama video
 - `PositioningSystem.py` - Define the function for point transformation and box transformation based on the `trans_params.json` files. It transforms the local coordinates of individual camera to the global coordinates.
 - `PositioningSystem_Test.py` - Test the positoning system of three farms by visualizing the panorama results of position transformation from each camera
 
 ## Usage
-- Video Stitching Test: Stitch the input videos to generate a panorama video:
+- Image Feature Extraction Test: Extract three kinds of features from the input image and visualize the result
+```
+    $ python feature_extraction_test.py
+```
+- Feature Matching Test: Match the features with BF/KNN methods. Select suitable matching method based on the Inliers Result
+```
+    $ python feature_matching_test.py
+```
+- Video Stitching Test: Stitch the input videos to generate a panorama video
 ```
     $ python stitch_custom.py -ivid /PATH/TO/VIDEO/GROUP -hpp /PATH/TO/HOMO/PARAMS/FILE -spp /PATH/TO/STITCH/PARAMS/FILE --farm_name FARM_NAME
 ```
-- Image Stitching Test: Stitch the images at the same frame from all cameras to generate a panorama image:
+- Image Stitching Test: Stitch the images at the same frame from all cameras to generate a panorama image
 ```
     $ python panorama_test.py
 ```

ROI_verify.py

@@ -5,7 +5,7 @@
 
 import utils
 from feature_extraction_test import Image
-import image_match_test as match_utils
+import feature_matching_test as match_utils
 
 if __name__ == '__main__':
 

image_match_test.py → feature_matching_test.py

@@ -3,120 +3,76 @@
 import cv2 as cv
 import itertools
 
-from feature_extraction_test import Image
-import utils
-
-def getMaskPointsInROIs(kps,ROIs):
-    """
-    Parameters
-    ----------
-    kps : List of feature elements
-    pts : n x 2 ndarray
-        Each row is a point (x,y)
-    ROIs : List of ROIs, each ROI is a size 4 iterable
-        Each ROI consists of (x1,y1,x2,y2), where (x1,y1) is the top left point
-        and (x2,y2) is the bottom right point
-
-    Returns
-    -------
-    ndarray of mask
-
-    """
-    pts = cv.KeyPoint_convert(kps)
-    submasks = []
-    for ROI in ROIs:
-        x_mask = np.logical_and(pts[:,0] >= ROI[0],pts[:,0] <= ROI[2])
-        y_mask = np.logical_and(pts[:,1] >= ROI[1],pts[:,1] <= ROI[3])
-        submasks.append(np.logical_and(x_mask,y_mask))
-
-    final_mask = np.zeros(submasks[0].shape,dtype=bool)
-    for mask in submasks:
-        final_mask = np.logical_or(final_mask,mask)
-
-    return final_mask
-
-def findHomography(matches, kps1, kps2):
-    queryIdxs = [match.queryIdx for match in matches]
-    trainIdxs = [match.trainIdx for match in matches]
-    kps2 = cv.KeyPoint_convert(kps2)
-    kps1 = cv.KeyPoint_convert(kps1)
-    homo_mat,inliers_mask = cv.findHomography(kps2[trainIdxs],kps1[queryIdxs],method=cv.RANSAC,ransacReprojThreshold=2)
-
-    return homo_mat, inliers_mask
-
+from stitch import Image, utils
 
 
 if __name__ == '__main__':
+    '''This script will be tested for feature matching'''
 
     draw_params = dict(matchColor = (0,255,0),
                    singlePointColor = (255,0,0),
                    flags = cv.DrawMatchesFlags_DEFAULT)
+
+    # Manual Set Range of Interests
+    ROIs1 = np.array([[350,150,760,1300]])
+    ROIs2 = np.array([[0,150,400,1300]])
 
     # load the matching images
-    img1 = cv.imread("lamp_15_distorted_empty.JPG")
-    img2 = cv.imread("lamp_14_distorted_empty.JPG")
+    img1 = cv.imread("dataset/Mathe/lamp_15_Mathe.PNG")
+    img2 = cv.imread("dataset/Mathe/lamp_14_Mathe.PNG")
 
     img1 = np.rot90(img1,1) 
     img2 = np.rot90(img2,1)
 
-    #img1 = utils.equalizeHist_old(img1)
-    #img2 = utils.equalizeHist_old(img2)
-
-    # Equalize histogram of images
+    # Initialize the Stitch Class
     Img1 = Image(img1)
     Img2 = Image(img2)
-
-    Img1.equalizeHist()
-    Img2.equalizeHist()
-
 
 
+    '''SIFT Features Matching Comparison'''
     # Find SIFT features of matching images
-    kps1_sift, dps1_sift = Img1.findFeatures('sift')
-    kps2_sift, dps2_sift = Img2.findFeatures('sift')
-
-    # Manual Set
-    # ROIs1 = np.array([[160,350,1250,760]])
-    # ROIs2 = np.array([[160,0,1250,400]])
-    ROIs1 = np.array([[350,150,760,1250]])
-    ROIs2 = np.array([[0,150,400,1250]])
-
-    final_mask1 = getMaskPointsInROIs(kps1_sift,ROIs1)
-    kps1_filter, des1_filter = Img1.featureFilter(final_mask1)
-
-    final_mask2 = getMaskPointsInROIs(kps2_sift,ROIs2)
-    kps2_filter, des2_filter = Img2.featureFilter(final_mask2)
+    kps1_sift, dps1_sift = Img1.kps, Img1.des
+    kps2_sift, dps2_sift = Img2.kps, Img2.des
+
+    # Filter the Features
+    masks1 = utils.getMaskPointsInROIs(kps1_sift, ROIs1)
+    masks2 = utils.getMaskPointsInROIs(kps2_sift, ROIs2)
 
+    kps1_filter, des1_filter = Img1.featureFilter(masks1[0])
+    kps2_filter, des2_filter = Img2.featureFilter(masks2[0])
 
+
     # BFMatches(des1_filter, des2_filter)
     matches_sift = utils.featureMatch(des1_filter, des2_filter, 'sift')
     matches_sift_knn = utils.featureMatch(des1_filter, des2_filter, 'sift', knn=True)
-    #bf_matches = FLANNMatches(des1_filter,des2_filter)
+
 
-    img_sift = cv.drawMatches(Img1.img,kps1_filter,Img2.img,kps2_filter,matches_sift[:50],None,**draw_params)
-    img_sift_knn = cv.drawMatches(Img1.img,kps1_filter,Img2.img,kps2_filter,matches_sift_knn,None,**draw_params)
-
-
+    img_sift = cv.drawMatches(Img1.img,kps1_filter,Img2.img,kps2_filter,matches_sift[:300],None,**draw_params)
+    img_sift_knn = cv.drawMatches(Img1.img,kps1_filter,Img2.img,kps2_filter,matches_sift_knn[:300],None,**draw_params)
 
+
+
+    '''BRISK Features Matching Comparison'''
+    # Extract the BRISK features
     kps1_brisk, dps1_brisk = Img1.findFeatures('brisk')
     kps2_brisk, dps2_brisk = Img2.findFeatures('brisk')
 
-    final_mask1_brisk = getMaskPointsInROIs(kps1_brisk,ROIs1)
-    kps1_filter_, des1_filter_ = Img1.featureFilter(final_mask1_brisk)
+    # Filter the Features
+    mask1_brisk = utils.getMaskPointsInROIs(kps1_brisk,ROIs1)
+    kps1_filter_, des1_filter_ = Img1.featureFilter(mask1_brisk[0])
 
-    final_mask2_brisk = getMaskPointsInROIs(kps2_brisk,ROIs2)
-    kps2_filter_, des2_filter_ = Img2.featureFilter(final_mask2_brisk)
-
+    mask2_brisk = utils.getMaskPointsInROIs(kps2_brisk,ROIs2)
+    kps2_filter_, des2_filter_ = Img2.featureFilter(mask2_brisk[0])
 
+    # BFMatches(des1_filter, des2_filter)
     matches_brisk = utils.featureMatch(des1_filter_, des2_filter_, 'brisk')
     matches_brisk_knn = utils.featureMatch(des1_filter_, des2_filter_, 'brisk',knn=True)
 
-
-
-    img_brisk = cv.drawMatches(Img1.img,kps1_filter_,Img2.img,kps2_filter_,matches_brisk[:50],None,**draw_params)
+    img_brisk = cv.drawMatches(Img1.img,kps1_filter_,Img2.img,kps2_filter_,matches_brisk[:300],None,**draw_params)
     img_brisk_knn = cv.drawMatches(Img1.img,kps1_filter_,Img2.img,kps2_filter_,matches_brisk_knn,None,**draw_params)
 
 
+    '''Visualize the Feature Matching Result'''
     plt.figure(1)
     plt.subplot(2,2,1)
     plt.title("Brute Force Matching on SIFT Features\n(# of Matches: %d)" %(len(matches_sift)))
@@ -137,20 +93,21 @@ def findHomography(matches, kps1, kps2):
     plt.title("Brute Force KNN Matching on BRISK Features\n(# of Matches: %d)" %(len(matches_brisk_knn)))
     plt.imshow(cv.cvtColor(img_brisk_knn, cv.COLOR_BGR2RGB))
     plt.axis('off')
+    plt.show()
 
 
-    homo_mat_sift, inliers_mask_sift = findHomography(matches_sift,kps1_filter,kps2_filter)
-    homo_mat_sift_knn, inliers_mask_sift_knn = findHomography(matches_sift_knn,kps1_filter,kps2_filter)
-    homo_mat_brisk, inliers_mask_brisk = findHomography(matches_brisk,kps1_filter_,kps2_filter_)
-    homo_mat_brisk_knn, inliers_mask_brisk_knn = findHomography(matches_brisk_knn,kps1_filter_,kps2_filter_)
-
+    '''Estimate the Inliers'''
+    homo_mat_sift, inliers_mask_sift = utils.findHomography(matches_sift,kps1_filter,kps2_filter)
+    homo_mat_sift_knn, inliers_mask_sift_knn = utils.findHomography(matches_sift_knn,kps1_filter,kps2_filter)
+    homo_mat_brisk, inliers_mask_brisk = utils.findHomography(matches_brisk,kps1_filter_,kps2_filter_)
+    homo_mat_brisk_knn, inliers_mask_brisk_knn = utils.findHomography(matches_brisk_knn,kps1_filter_,kps2_filter_)
 
     matches_inliers_sift = list(itertools.compress(matches_sift, inliers_mask_sift))
     matches_inliers_sift_knn = list(itertools.compress(matches_sift_knn, inliers_mask_sift_knn))
     matches_inliers_brisk = list(itertools.compress(matches_brisk, inliers_mask_brisk))
     matches_inliers_brisk_knn = list(itertools.compress(matches_brisk_knn, inliers_mask_brisk_knn))
 
-
+    '''Draw the Inliers'''
     img_inliners_sift = cv.drawMatches(Img1.img,kps1_filter,Img2.img,kps2_filter,matches_inliers_sift,None,**draw_params)
     img_inliners_sift_knn = cv.drawMatches(Img1.img,kps1_filter,Img2.img,kps2_filter,matches_inliers_sift_knn,None,**draw_params)
     img_inliners_brisk = cv.drawMatches(Img1.img,kps1_filter_,Img2.img,kps2_filter_,matches_inliers_brisk,None,**draw_params)

stitch.py

@@ -5,7 +5,7 @@
 
 import utils
 from feature_extraction_test import Image
-import image_match_test as match_utils
+import feature_matching_test as match_utils
 
 # Define the draw parameters for matching visualization
 draw_params = dict(matchColor = (0,255,0),

stitch/ImageStitch.py

@@ -5,24 +5,31 @@
 
 '''
 IMAGE Class: 
+
 Define the feature properties for single image
 ----------
+self.img : Camera Image from Single Frame [np.array]
+self.nfeatures : Number of Total Features [int]
+self.kps : Total Key Points (Features) [tuple]
+self.des : Total Key Points Descriptors [np.array]
+self.kpsCluster : Cluster List of Key Points(Features) [list of tuple]
+self.desCluster : Cluster List of Descriptors [list of np.array]
 '''
 class Image(object):
     # Initialization for the Image object
     def __init__(self, img, nfeatures=0):
         self.img = img
         self.nfeatures = nfeatures
-        self.kps = None
+        self.kps = None # Position of Total Features 
         self.des = None
         self.kpsCluster = None
         self.desCluster = None
 
         # Equalize the YUV channels histogram
-        #self.equalizeHist()
+        self.equalizeHist()
 
         # Extract the features from image and update the feature property
-        kps,des = self.findFeatures('sift')
+        kps, des = self.findFeatures('sift')
         print("\nInitial Successfullys")
 
     def equalizeHist(self):
@@ -71,6 +78,7 @@ def featureCluster(self, masks):
         self.kpsCluster = kpsCluster
         self.desCluster = desCluster
 
+
 
 '''
 STITCH CLASS: 
@@ -93,7 +101,7 @@ def featureExtract(self, ROIs1, ROIs2):
 
     def homoEstimate(self):
         # Define the matches based on two images
-        matches_list = utils.clusterMatch(self.Img1.desCluster, self.Img2.desCluster)
+        matches_list = self.clusterMatch()
 
         # Combine the features in one lists from each cluster
         self.featureIntegrate(matches_list)
@@ -103,6 +111,25 @@ def homoEstimate(self):
 
         return homo_mat, matches_inliers
 
+
+    def clusterMatch(self):
+        matches_list = []
+        for i in range(len(self.Img1.desCluster)):
+            des1 = self.Img1.desClusterdesCluster[i]
+            des2 = self.Img2.desClusterdesCluster[i]
+
+            bf = cv.BFMatcher()
+
+            match = bf.knnMatch(des1, des2, k=3)
+
+            matchFilter = []
+            for m, _, n in match:
+                if m.distance < 0.9 * n.distance:
+                    matchFilter.append(m)
+            matches_list.append(matchFilter)
+        return matches_list
+
+
     def featureIntegrate(self, matches_list):
         # Define the variables
         kpsCluster1 = self.Img1.kpsCluster

stitch/utils.py

@@ -4,6 +4,8 @@
 import math
 import matplotlib.pyplot as plt
 
+'''Image Class Functions'''
+
 def equalizeHist(img):
     img_yuv = cv.cvtColor(img, cv.COLOR_BGR2YUV)
     img_yuv[:, :, 0] = cv.equalizeHist(img_yuv[:, :, 0])
@@ -79,33 +81,14 @@ def featureMatch(des1, des2, method, knn=False):
         matches = bf.match(des1, des2)
         matches_good = sorted(matches, key=lambda x: x.distance)
     else:
-        matches = bf.knnMatch(des1, des2, k=2)
+        matches = bf.knnMatch(des1, des2, k=3)
 
         matches_good = []
-        for m, n in matches:
-            if m.distance < 0.8*n.distance:
+        for m,_,n in matches:
+            if m.distance < 0.9*n.distance:
                 matches_good.append(m)
     return matches_good
 
-
-def clusterMatch(desCluster1, desCluster2):
-    matches = []
-    for i in range(len(desCluster1)):
-        des1 = desCluster1[i]
-        des2 = desCluster2[i]
-
-        bf = cv.BFMatcher()
-
-        match = bf.knnMatch(des1, des2, k=3)
-
-        matchFilter = []
-        for m, _, n in match:
-            if m.distance < 0.9 * n.distance:
-                matchFilter.append(m)
-        matches.append(matchFilter)
-    return matches
-
-
 def findHomography(matches, kps1, kps2):
     queryIdxs = [match.queryIdx for match in matches]
     trainIdxs = [match.trainIdx for match in matches]

undistortion.py

@@ -9,7 +9,7 @@
 
 import utils
 from stitch.ImageStitch import Stitch
-import image_match_test as match_utils
+import feature_matching_test as match_utils
 import feature_mapping