Skip to content

Image search easyocr one #375

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Oct 17, 2023
Merged

Image search easyocr one #375

Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
2edf83f
fixed easy ocr issue as it was not wroking , works now.
nadimurrashid Aug 24, 2023
a24263b
you can now save easy ocr coordinates inside a variable and use that …
nadimurrashid Aug 25, 2023
becfaf8
Did slight modification to make it more faster with the variable
nadimurrashid Aug 29, 2023
735bdcf
remove some older code from older function
nadimurrashid Oct 17, 2023
e741d97
remove some older code from older function
nadimurrashid Oct 17, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
228 changes: 80 additions & 148 deletions Framework/Built_In_Automation/Desktop/Windows/BuiltInFunctions.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@
#########################
import pdb

global Shared_Resources

code_debug = False
tabs = 0
import sys, os, subprocess
Expand Down Expand Up @@ -34,6 +36,7 @@
from PIL import Image, ImageGrab



python_folder = []
for location in subprocess.getoutput("where python").split("\n"):
if "Microsoft" not in location:
Expand Down Expand Up @@ -1092,11 +1095,7 @@ def image_search(step_data_set):
left_width = ""
top_height = ""
colour_state = ""
method_image = ""
language = ""
t_conf = 0.9
text_screenshot = ''
easyocr_paragraph = ''


for left, mid, right in step_data_set:
left = left.strip().lower()
Expand All @@ -1116,16 +1115,7 @@ def image_search(step_data_set):
top_height = float(right)
elif "colour" in left:
colour_state = right
elif 'language' in left:
language = right
elif 'method_image' in left:
method_image = right
elif 't_conf' in left:
t_conf = float(right)
elif 't_screenshot' in left:
text_screenshot = right
elif 'easyocr_paragraph' in left:
easyocr_paragraph = right


else:
if not image_text:
Expand All @@ -1142,10 +1132,7 @@ def image_search(step_data_set):
top_height = 1
if colour_state == '':
colour_state = 'black_white'
if method_image == '':
method = 'method_1'
if language == '':
language = 'en'


if parent_dataset:
parent = Get_Element(parent_dataset)
Expand Down Expand Up @@ -1177,129 +1164,54 @@ def image_search(step_data_set):
try:
if image_text:

if method_image == 'method_1':
import cv2
from pytesseract import pytesseract
pytesseract.tesseract_cmd = os.environ["PROGRAMFILES"] + r"\Tesseract-OCR\tesseract.exe"

image_text = image_text.replace(" ", "").lower()
PIL.ImageGrab.grab().crop((left, top, left + width * left_width, top + height * top_height)).save("sample.jpg")
imge = cv2.imread("sample.jpg")
gray = cv2.cvtColor(imge, cv2.COLOR_BGR2GRAY)

if colour_state == "black_white":
data = pytesseract.image_to_boxes(gray)
else:
data = pytesseract.image_to_boxes(imge)
all_letters = data.split("\n")
print(all_letters)
full_string = ""
for i in all_letters:
full_string += i[0] if len(i) > 0 else "~"
full_string = full_string.lower()

all_pos = [m.start() for m in re.finditer(image_text, full_string)]

if -len(all_pos) <= idx < len(all_pos):
CommonUtil.ExecLog(sModuleInfo, "Found %s text elements. Returning element of index %s" % (len(all_pos), idx), 1)
i = all_pos[idx]
elif len(all_pos) != 0:
CommonUtil.ExecLog(sModuleInfo, "Found %s text elements. Index out of range" % len(all_pos), 3)
return "zeuz_failed"
else:
CommonUtil.ExecLog(sModuleInfo, 'Could not find text "%s"' % image_text, 3)
return "zeuz_failed"

msg = ""
a = all_letters[i:i + len(image_text)]
for i in a:
msg += i + "\n"
left_top = list(map(int, a[0].split(" ")[1:3]))
right_bottom = list(map(int, a[-1].split(" ")[3:5]))
center = left + (right_bottom[0] + left_top[0]) // 2, top + height - (right_bottom[1] + left_top[1]) // 2
msg += "Center = " + str(center) + "\n"
# pyautogui.moveTo(center)

element = left_top[0] + left, height - right_bottom[1] + top, right_bottom[0] - left_top[0], right_bottom[1] - left_top[1]
msg += "Coordinates = " + str(element) + "\n"
CommonUtil.ExecLog(sModuleInfo, msg, 5)
import cv2
from pytesseract import pytesseract
pytesseract.tesseract_cmd = os.environ["PROGRAMFILES"] + r"\Tesseract-OCR\tesseract.exe"

return _Element(element)
image_text = image_text.replace(" ", "").lower()
PIL.ImageGrab.grab().crop((left, top, left + width * left_width, top + height * top_height)).save("sample.jpg")
imge = cv2.imread("sample.jpg")
gray = cv2.cvtColor(imge, cv2.COLOR_BGR2GRAY)

if colour_state == "black_white":
data = pytesseract.image_to_boxes(gray)
else:
data = pytesseract.image_to_boxes(imge)
all_letters = data.split("\n")
print(all_letters)
full_string = ""
for i in all_letters:
full_string += i[0] if len(i) > 0 else "~"
full_string = full_string.lower()

all_pos = [m.start() for m in re.finditer(image_text, full_string)]

if -len(all_pos) <= idx < len(all_pos):
CommonUtil.ExecLog(sModuleInfo, "Found %s text elements. Returning element of index %s" % (len(all_pos), idx), 1)
i = all_pos[idx]
elif len(all_pos) != 0:
CommonUtil.ExecLog(sModuleInfo, "Found %s text elements. Index out of range" % len(all_pos), 3)
return "zeuz_failed"
else:
import easyocr
import numpy as np
import cv2
from pytesseract import pytesseract
from difflib import SequenceMatcher
CommonUtil.ExecLog(sModuleInfo, 'Could not find text "%s"' % image_text, 3)
return "zeuz_failed"

pytesseract.tesseract_cmd = os.environ["PROGRAMFILES"] + r"\Tesseract-OCR\tesseract.exe"
msg = ""
a = all_letters[i:i + len(image_text)]
for i in a:
msg += i + "\n"
left_top = list(map(int, a[0].split(" ")[1:3]))
right_bottom = list(map(int, a[-1].split(" ")[3:5]))
center = left + (right_bottom[0] + left_top[0]) // 2, top + height - (right_bottom[1] + left_top[1]) // 2
msg += "Center = " + str(center) + "\n"
# pyautogui.moveTo(center)

PIL.ImageGrab.grab().save("sample.png")
reader = easyocr.Reader([language])
if easyocr_paragraph == 'true':
output = reader.readtext("sample.png",paragraph=True)
else:
output = reader.readtext("sample.png", paragraph=False)

item = []
count = 0
crop_counter = 0
for text in output:
def seq(a, b):
c = SequenceMatcher(a=a, b=b).ratio()
print(c)
if c > 0.8:
return c
element = left_top[0] + left, height - right_bottom[1] + top, right_bottom[0] - left_top[0], right_bottom[1] - left_top[1]
msg += "Coordinates = " + str(element) + "\n"
CommonUtil.ExecLog(sModuleInfo, msg, 5)

return _Element(element)

else:
return .00004

rslt = seq(image_text, text[1])
if rslt >= float(t_conf):
# if image_text in text[1]:
item.append([text])
print(text)
CommonUtil.ExecLog(sModuleInfo, "Found %s text. Returning element of index %s" % (image_text, count), 1)
count = count + 1
else:
print(text)
continue

if item == []:
CommonUtil.ExecLog(sModuleInfo, 'Could not find text "%s"' % image_text, 3)
return "zeuz_failed"
cord = np.array(item[idx][0])
cord1 = cord.tolist()
#
x_min, y_min = [min(cord_val) for cord_val in zip(*cord1[0])]
x_max, y_max = [max(cord_val) for cord_val in zip(*cord1[0])]

if text_screenshot !='':
img = cv2.imread("sample.png")
cropping = img[y_min:y_max , x_min:x_max]
cv2.imwrite('cropped_image.png', cropping)
from PIL import Image

image = Image.open('cropped_image.png')
gray_image = image.convert('L')
output_2 = pytesseract.image_to_string(gray_image)
print(output_2)

if output_2 in image_text or image_text in output_2:
element = x_min, y_min, x_max - x_min, y_max - y_min
crop_counter = 1
return _Element(element)
else:
CommonUtil.ExecLog(sModuleInfo, 'Could not find text "%s"' % image_text, 3)
return "zeuz_failed"
else:
pass
if crop_counter == 0:
element = x_min,y_min,x_max-x_min,y_max-y_min
return _Element(element)
else:
pass

else:
# Scale image if required
Expand Down Expand Up @@ -1338,7 +1250,6 @@ def seq(a, b):

@logger
def new_image_text(step_data_set):
install_ocr()
sModuleInfo = inspect.currentframe().f_code.co_name + " : " + MODULE_NAME

try:
Expand All @@ -1355,14 +1266,16 @@ def new_image_text(step_data_set):
t_conf = 0.9
text_screenshot = ''
easyocr_paragraph = ''
previous_stored_data = None
store_data = ''

for left, mid, right in step_data_set:
left = left.strip().lower()
mid = mid.strip().lower()
if mid == "element parameter":
if "index" in left:
idx = int(right.strip())
elif "new_image_text" in left:
elif "easytxt" in left:
image_text = right
elif 'language' in left:
language = right
Expand All @@ -1372,13 +1285,26 @@ def new_image_text(step_data_set):
text_screenshot = right
elif 'easyocr_paragraph' in left:
easyocr_paragraph = right
elif "store_data" in left:
store_data = right.strip()
elif 'previous_value' in left:
previous_stored_data = CommonUtil.parse_value_into_object(right.strip())

if language == '':
language = 'en'
PIL.ImageGrab.grab().save("sample.png")
reader = easyocr.Reader([language])
if easyocr_paragraph == 'true':
output = reader.readtext("sample.png", paragraph=True)
if previous_stored_data == None:
if easyocr_paragraph == 'true':
reader = easyocr.Reader([language])
output = reader.readtext("sample.png", paragraph=True)
Shared_Resources.Set_Shared_Variables(store_data, output)
else:
reader = easyocr.Reader([language])
output = reader.readtext("sample.png", paragraph=False)
Shared_Resources.Set_Shared_Variables(store_data, output)
else:
output = reader.readtext("sample.png", paragraph=False)
output = previous_stored_data


item = []
count = 0
Expand Down Expand Up @@ -1497,7 +1423,7 @@ def Get_Element(data_set, wait_time=Shared_Resources.Get_Shared_Variables("eleme
elif "automation" in left: element_automation = [right, _count_star(left)] # automationid
elif "control" in left: element_control = [right, _count_star(left)] # localizedcontroltype
elif "path" in left: element_path = right.strip()
elif "new_image_text" in left:
elif "easytxt" in left:
element_image.append((left, mid, right))
elif "image" in left:
element_image.append((left, mid, right))
Expand Down Expand Up @@ -1553,11 +1479,12 @@ def Get_Element(data_set, wait_time=Shared_Resources.Get_Shared_Variables("eleme
left_width = 1
top_height = 1
colour_state = 'black_white'
method_image = 'method_1'
t_conf = 0.9
language = 'en'
text_screenshot = ''
easyocr_paragraph = ''
previous_stored_data = None
store_data = ''

for left, mid, right in data_set:
left = left.strip().lower()
Expand All @@ -1571,14 +1498,16 @@ def Get_Element(data_set, wait_time=Shared_Resources.Get_Shared_Variables("eleme
colour_state = right
elif 'language' in left:
language = right
elif 'method_image' in left:
method_image = right
elif 't_conf' in left:
t_conf = right
elif 't_screenshot' in left:
text_screenshot = right
elif 'easyocr_paragraph' in left:
easyocr_paragraph = right
elif 'previous_value' in left:
previous_stored_data = CommonUtil.parse_value_into_object(right.strip())
elif 'store_data' in left:
store_data = right.strip()

_get_main_window(window_name)
for i in (("name", parent_name), ("class", parent_class), ("automationid", parent_automation), ("control", parent_control), ("path", parent_path), ("window", window_name), ("index", parent_index)):
Expand All @@ -1593,12 +1522,15 @@ def Get_Element(data_set, wait_time=Shared_Resources.Get_Shared_Variables("eleme
element_image.append(("top_height", "element parameter", str(top_height)))
element_image.append(("colour_state", "element parameter", str(colour_state)))
element_image.append(("language", "element parameter", str(language)))
element_image.append(("method_image", "element parameter", str(method_image)))
element_image.append(("t_conf", "element parameter", str(t_conf)))
element_image.append(("t_screenshot", "element parameter", str(text_screenshot)))
element_image.append(("easyocr_paragraph", "element parameter", str(easyocr_paragraph)))
element_image.append(("previous_value", "element parameter", str(previous_stored_data)))
element_image.append(("store_data", "element parameter", str(store_data)))



if 'new_image_text' in element_image[0][0]:
if 'easytxt' in element_image[0][0]:
result = new_image_text(element_image)
return result
else:
Expand Down