Skip to content

Commit

Permalink
Solved overlapping hard boxes
Browse files Browse the repository at this point in the history
  • Loading branch information
@gpisanelli
gpisanelli committed Mar 24, 2020
1 parent b7589f1 commit 0161987
Show file tree
Hide file tree
Showing 4 changed files with 94 additions and 100 deletions.
83 changes: 3 additions & 80 deletions src/main.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
scene_dict = {
'-e': ['e1.png', 'e2.png', 'e3.png', 'e4.png', 'e5.png'],
'-m': ['m1.png', 'm2.png', 'm3.png', 'm4.png', 'm5.png'],
'-h': ['h1.jpg']
'-h': ['h1.jpg', 'h2.jpg', 'h3.jpg', 'h4.jpg', 'h5.jpg']
}

# Image loading
Expand Down Expand Up @@ -242,6 +242,7 @@ def main():
bar_intersected = bounds_dict[gray_index]
bar_intersecting = (bounds, test_box, box_name)
result = object_validation.compare_detections(bar_intersected, bar_intersecting)
# if result == 1 allora l'intersecato è interno all'intersecante, non faccio nulla
if result == 0: # devo sostituire nel dict e nella mask l'intersecato con l'intersecante
# per cancellare dalla mask l'intersecato disegno sul suo baricentro un baricentro nero delle stesse dimensioni
M_intersected = cv2.moments(bar_intersected[0])
Expand All @@ -256,92 +257,14 @@ def main():
cv2.circle(matches_mask, (cx, cy), w // 4, color, -1)
bounds_dict[color] = bar_intersecting
color += 1
else: # niente intersezione, aggiungo al dict
else: # niente intersezione, aggiungo al dict e alla mask
cv2.circle(matches_mask, (cx, cy), w // 4, color, -1)
bounds_dict[color] = (bounds, test_box, box_name)
color += 1
else:
print('Box {} failed color validation'.format(box_name))
else:
print('Box {} failed convex validation'.format(box_name))
'''
caso in cui non ho abbastanza keypoint per la homography, non dovrebbe servire e anyway non funge molto bene
elif len(good_matches[i][1]) > 0:
# disegno rettangolo approssimativo perché not enough points per la homography
# cerco la miglior scale e rotation facendo la media tra quelle disponibili
scales = []
rotations = []
for j in range(len(good_matches[i][1])):
scales.append(kp_s[good_matches[i][1][j].trainIdx].size / kp_t[good_matches[i][1][j].queryIdx].size)
rotations.append(kp_s[good_matches[i][1][j].trainIdx].angle - kp_t[good_matches[i][1][j].queryIdx].angle)
scale_factor = np.mean(scales)
rotation_factor = np.mean(rotations)
print('Scale mean {} = {}'.format(i, scale_factor))
print('Rotation mean {} = {}'.format(i, rotation_factor))
# computo i vettori congiungenti il centro del template ai suoi vertici (ordine importante per polylines)
box_vertexes = [(0, 0), (proc_box.shape[1], 0), (proc_box.shape[1], proc_box.shape[0]), (0, proc_box.shape[0])]
scene_vertexes = []
vectors = []
int_barycenter = np.int32(barycenter)
for j in range(4):
vectors.append(np.subtract(box_vertexes[j], int_barycenter))
# print('Box vertexes {} = {}'.format(i, box_vertexes))
# print('Box vectors {} = {}'.format(i, vectors))
template_copy = proc_box.copy()
for j in range(4):
cv2.circle(template_copy, box_vertexes[j], 20, (0, 255, 0), -1)
cv2.circle(template_copy, (int(barycenter[0]), int(barycenter[1])), 20, (0, 255, 0), -1)
cv2.arrowedLine(template_copy,
(int(barycenter[0]), int(barycenter[1])),
(int(barycenter[0] + vectors[j][0]), int(barycenter[1] + vectors[j][1])), (0, 255, 0))
visualization.display_img(template_copy, title='Template_vertexes_vectors')
# scalo e ruoto i vettori secondo le scale e rotation prima trovate
for j in range(4):
vectors[j] = np.multiply(vectors[j], scale_factor)
vectors[j] = np.reshape(vectors[j], (2, 1))
vectors[j] = rotate(vectors[j], math.radians(rotation_factor))
vectors[j] = vectors[j].reshape(2)
cv2.arrowedLine(template_copy,
(int(barycenter[0]), int(barycenter[1])),
(int(barycenter[0] + vectors[j][0]), int(barycenter[1] + vectors[j][1])), (0, 255, 0))
# computo i vertici risultanti nella scena
scene_vertex_x = int(round(good_matches[i][0][0] + vectors[j][0]))
scene_vertex_y = int(round(good_matches[i][0][1] + vectors[j][1]))
scene_vertexes.append((scene_vertex_x, scene_vertex_y))
# print('Scene vertexes match {} = {}'.format(i, scene_vertexes))
scene_copy = scene.copy()
for j in range(4):
cv2.circle(scene_copy, (int(good_matches[i][0][0]), int(good_matches[i][0][1])), 10, (0, 255, 0), -1)
cv2.circle(scene_copy, scene_vertexes[j], 10, (0, 255, 0), -1)
cv2.arrowedLine(scene_copy, scene_vertexes[j], (int(round(scene_vertexes[j][0]-vectors[j][0])),
int(round(scene_vertexes[j][1]-vectors[j][1]))), (0, 255, 0))
visualization.display_img(scene_copy, title='Scene_vertexes_vectors')
# Object validation
polygon_convex = object_validation.is_convex_polygon(scene_vertexes)
if polygon_convex:
# homography sarebbe da computare tra gli 8 punti vertici del box e della scena, gli used_points sono quei vertici
color_validation = object_validation.validate_color(test_box, test_scene, used_src_points,
used_dst_points, scene_vertexes, homography)
if color_validation:
visualization_scene = visualization.draw_polygons(visualization_scene, [scene_vertexes])
visualization_scene = visualization.draw_names(visualization_scene, scene_vertexes, box_name)
else:
print('Box {} failed color validation'.format(box_name))
else:
print('Box {} failed convex validation'.format(box_name))
visualization_scene = visualization.draw_polygons(visualization_scene, np.int32([scene_vertexes]))
'''

for key in bounds_dict:
visualization_scene = visualization.draw_polygons(visualization_scene, [ bounds_dict[key][0] ])
visualization_scene = visualization.draw_names(visualization_scene, bounds_dict[key][0], bounds_dict[key][2])
Expand Down
45 changes: 43 additions & 2 deletions src/object_validation.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import image_processing
import visualization
import matplotlib.pyplot as plt
import math

HISTOGRAM_THRESHOLD = 1.5

Expand All @@ -15,12 +16,15 @@ def is_convex_polygon(bounds):
return cv2.isContourConvex(bounds)


def check_rectangularity(bounds):
def compute_rectangularity(bounds):
x, y, w, h = cv2.boundingRect(bounds)
bounds_area = cv2.contourArea(bounds)
rect_area = w * h
return bounds_area / rect_area


return bounds_area / rect_area > 0.9
def check_rectangularity(bounds):
return compute_rectangularity(bounds) > 0.85


def is_contained(bounds_outer, bounds_inner):
Expand All @@ -45,6 +49,43 @@ def compare_detections(intersected, intersecting):
return -1


def is_contained_hard(bounds_outer, bounds_inner, outer_scaling):
x_o, y_o, w_o, h_o = cv2.boundingRect(bounds_outer)
x_i, y_i, w_i, h_i = cv2.boundingRect(bounds_inner)

# con le operazioni successive scalo (x_o, y_o), w_o, h_o in modo che definiscano un rettangolo con area
# ingrandita in base ad outer_scaling (ad esempio con 1.5 aumenta del 50%)
middle_x, middle_y = x_o + (w_o / 2), y_o + (h_o / 2)
w_o, h_o = w_o * math.sqrt(outer_scaling), h_o * math.sqrt(outer_scaling)
x_o, y_o = middle_x - (w_o / 2), middle_y - (h_o / 2)

ul = (x_o, y_o) <= (x_i, y_i)
bl = x_o <= x_i and y_o + h_o >= y_i + h_i
br = (x_o + w_o, y_o + h_o) >= (x_i + w_i, y_i + h_i)
ur = x_o + w_o >= x_i + w_i and y_o <= y_i

return bl and ul and ur and br


def find_best(bounds1, bounds2):
rectangularity1 = compute_rectangularity(bounds1)
rectangularity2 = compute_rectangularity(bounds2)
if rectangularity1 >= rectangularity2:
return 1
else:
return 0


# ritorna 1 se devo mantenere intersected, 0 se devo mantenere intersecting, -1 se devo mantenere entrambi
def compare_detections_hard(intersected, intersecting, outer_scaling=1):
contained2in1 = is_contained_hard(intersected[0], intersecting[0], outer_scaling)
contained1in2 = is_contained_hard(intersecting[0], intersected[0], outer_scaling) # probabilmente useless, i box dovrebbero essere così simili che è indifferente quale ingrandisco
if contained2in1 or contained1in2: # due box sono troppo sovrapposti, mantengo solo il migliore
return find_best(intersected[0], intersecting[0])
else: # c'è intersezione ma i box non sono troppo sovrapposti
return -1


# Draws a black rectangle around each matching point found in the two images, obtaining a box and a scene in which only
# the regions of the image that contain no important features are left. Because of how the feature detection algorithm
# works, features belong to areas with edges and intensity variations. In the case of grocery products, regions with
Expand Down
57 changes: 40 additions & 17 deletions src/parallel_hough.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ def split_shelves(scene):
for row in mean_lines:
cv2.line(mean_lines_img, (0, row), (mean_lines_img.shape[1], row), (255, 0, 0), 1)

visualization.display_img(mean_lines_img)
#visualization.display_img(mean_lines_img)

sub_images = []
sub_images.append((scene[0:mean_lines[0], 0:scene.shape[1]], 0))
Expand Down Expand Up @@ -101,6 +101,15 @@ def _pickle_keypoints(point):
point.response, point.octave, point.class_id)


def erase_bar(matches_mask, bounds):
# per cancellare dalla mask il baricentro disegno sopra un baricentro nero delle stesse dimensioni
M_intersected = cv2.moments(bounds)
cx_intersected = int(M_intersected['m10'] / M_intersected['m00'])
cy_intersected = int(M_intersected['m01'] / M_intersected['m00'])
_, _, w_intersected, _ = cv2.boundingRect(bounds)
cv2.circle(matches_mask, (cx_intersected, cy_intersected), w_intersected // 4, (0, 0, 0), -1)


def compute_sub_image(dict_box_features, sub_image):
sub_scene, y = sub_image
proc_scene = preprocess_sub_scene(sub_scene)
Expand All @@ -112,7 +121,7 @@ def compute_sub_image(dict_box_features, sub_image):
found_bounds = {}

matches_mask = np.zeros(proc_scene.shape, dtype=np.uint8)
color = 0
color = 1
bounds_dict = {}

for box_name in dict_box_features:
Expand All @@ -137,26 +146,39 @@ def compute_sub_image(dict_box_features, sub_image):
intersection = cv2.bitwise_and(new_bar_copy, matches_mask)

if cv2.countNonZero(intersection) > 0:
print('\n\nIntersection\n\n')
#print('\n\nIntersection between the following boxes')
gray_index = intersection[intersection > 0][0]
bar_intersected = bounds_dict[gray_index]
bar_intersecting = (b, box, box_name)
'''
print('Name intersected ', bar_intersected[2])
print('Name intersecting ', bar_intersecting[2])
test = sub_scene.copy()
test = visualization.draw_polygons(test, [bar_intersected[0]])
test = visualization.draw_polygons(test, [bar_intersecting[0]])
visualization.display_img(test)
'''
result = object_validation.compare_detections(bar_intersected, bar_intersecting)
#print('Result of compare_detections = ', result)
#if result == 1 allora l'intersecato è interno all'intersecante, non faccio nulla
if result == 0: # devo sostituire nel dict e nella mask l'intersecato con l'intersecante
# per cancellare dalla mask l'intersecato disegno sul suo baricentro un baricentro nero delle stesse dimensioni
M_intersected = cv2.moments(bar_intersected[0])
cx_intersected = int(M_intersected['m10'] / M_intersected['m00'])
cy_intersected = int(M_intersected['m01'] / M_intersected['m00'])
_, _, w_intersected, _ = cv2.boundingRect(bar_intersected[0])
cv2.circle(matches_mask, (cx_intersected, cy_intersected), w_intersected // 4, (0, 0, 0), -1)

erase_bar(matches_mask, bar_intersected[0])
cv2.circle(matches_mask, (cx, cy), w // 4, color, -1)
bounds_dict[gray_index] = bar_intersecting
if result == -1: # c'è intersezione ma i box non sono uno dentro l'altro, aggiungo l'intersecante al dict e alla mask
cv2.circle(matches_mask, (cx, cy), w // 4, color, -1)
bounds_dict[color] = bar_intersecting
color += 1
else: # niente intersezione, aggiungo al dict
if result == -1: # c'è intersezione ma i box non sono completamente uno dentro l'altro, effettuo ulteriore controllo con compare_detections_hard
#print('Calling compare_detections_hard')
result = object_validation.compare_detections_hard(bar_intersected, bar_intersecting, 1.5)
#print('Result of compare_detections_hard = ', result)
# if result == 1 allora c'è troppa sovrapposizione tra i box e l'intersecato è migliore dell'intersecante, non faccio nulla
if result == 0: # c'è troppa sovrapposizione tra i box e l'intersecante è migliore dell'intersecato, sostituisco
erase_bar(matches_mask, bar_intersected[0])
cv2.circle(matches_mask, (cx, cy), w // 4, color, -1)
bounds_dict[gray_index] = bar_intersecting
if result == -1: # anche secondo compare_detections_hard c'è intersezione ma non sovrapposizione, aggiungo al dict e alla mask
cv2.circle(matches_mask, (cx, cy), w // 4, color, -1)
bounds_dict[color] = bar_intersecting
color += 1
else: # niente intersezione, aggiungo al dict e alla mask
cv2.circle(matches_mask, (cx, cy), w // 4, color, -1)
bounds_dict[color] = (b, box, box_name)
color += 1
Expand All @@ -172,7 +194,8 @@ def compute_sub_image(dict_box_features, sub_image):
visualization_scene = visualization.draw_polygons(visualization_scene, [bounds_dict[key][0]])
visualization_scene = visualization.draw_names(visualization_scene, bounds_dict[key][0], bounds_dict[key][2])

visualization.display_img(visualization_scene)
#visualization.display_img(visualization_scene)
#cv2.imwrite('/home/ginetto/Desktop/' + str(time.time()) + '.jpg', visualization_scene)

return found_bounds

Expand All @@ -187,6 +210,6 @@ def poolcontext(*args, **kwargs):
def hough_sub_images(sub_images, dict_template_features):
copyreg.pickle(cv2.KeyPoint().__class__, _pickle_keypoints)
with poolcontext(processes=1) as pool:
results = pool.map(partial(compute_sub_image, dict_template_features), sub_images[3:4])
results = pool.map(partial(compute_sub_image, dict_template_features), sub_images)

return results
9 changes: 8 additions & 1 deletion src/visualization.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import platform
import subprocess
import matplotlib.pyplot as plt
import random
import load_images

if platform.system() == 'Windows':
Expand Down Expand Up @@ -57,6 +58,12 @@ def correct_if_oversized(img):
def oversized_bool_list(img):
return [img.shape[0] > screen_height, img.shape[1] > screen_width]

'''
def random_rgb():
rgbl = [255, 0, 0]
random.shuffle(rgbl)
return tuple(rgbl)
'''

def draw_polygons(image, polygons):
img_copy = image.copy()
Expand All @@ -76,5 +83,5 @@ def draw_names(img, bounds, box_name):
thickness = int(1.5 * img.shape[0] * img.shape[1] / (800*800))
rec_mask = cv2.rectangle(rec_mask, (bounds[0][0] - 5, int((bounds[0][1] + bounds[1][1]) / 2 - 35*fontScale)), (int(bounds[0][0] + len(name)*20*fontScale), int((bounds[0][1] + bounds[1][1]) / 2 + 15*fontScale)), (255,255,255), thickness=-1)
img = cv2.addWeighted(img, 1, rec_mask, 0.2, 0)
img = cv2.putText(img, name, (bounds[0][0] + 5, int((bounds[0][1] + bounds[1][1]) / 2)), fontFace, fontScale, (0,0,0), thickness)
img = cv2.putText(img, name, (bounds[0][0] + 5, int((bounds[0][1] + bounds[1][1]) / 2)), fontFace, fontScale, (0, 0, 0), thickness)
return img

0 comments on commit 0161987

Please sign in to comment.