Skip to content

Commit 558e3a4

Browse files
aqualityAutomationaqualityAutomation
authored
[Feature] Find elements by image (#123) +semver: feature
Implement ByImage locator. Location strategy: - Finding the closest element to matching point instead of the topmost element/ all elements on point - Support finding multiple elements (multiple image matches) - Support relative search (e.g. from element) - Add javadocs - Add js script to getElementsFromPoint - Add locator test
2 parents 9445843 + eb6dde5 commit 558e3a4

File tree

9 files changed

+260
-2
lines changed

9 files changed

+260
-2
lines changed

‎pom.xml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,12 @@
100100
<version>2.0.10</version>
101101
<scope>test</scope>
102102
</dependency>
103+
<dependency>
104+
<groupId>org.openpnp</groupId>
105+
<artifactId>opencv</artifactId>
106+
<version>[4.7.0,)</version>
107+
</dependency>
108+
103109
<dependency>
104110
<groupId>org.testng</groupId>
105111
<artifactId>testng</artifactId>

‎src/main/java/aquality/selenium/browser/JavaScript.java

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,9 @@ public enum JavaScript {
2121
GET_CHECKBOX_STATE("getCheckBxState.js"),
2222
GET_COMBOBOX_SELECTED_TEXT("getCmbText.js"),
2323
GET_COMBOBOX_TEXTS("getCmbValues.js"),
24+
GET_DEVICE_PIXEL_RATIO("getDevicePixelRatio.js"),
2425
GET_ELEMENT_BY_XPATH("getElementByXpath.js"),
26+
GET_ELEMENTS_FROM_POINT("getElementsFromPoint.js"),
2527
GET_ELEMENT_CSS_SELECTOR("getElementCssSelector.js"),
2628
GET_ELEMENT_XPATH("getElementXPath.js"),
2729
GET_ELEMENT_TEXT("getElementText.js"),

‎src/main/java/aquality/selenium/elements/interfaces/ByImage.java

Lines changed: 180 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,180 @@
1+
package aquality.selenium.elements.interfaces;
2+
3+
import aquality.selenium.browser.AqualityServices;
4+
import aquality.selenium.browser.JavaScript;
5+
import org.opencv.core.Point;
6+
import org.opencv.core.*;
7+
import org.opencv.imgcodecs.Imgcodecs;
8+
import org.opencv.imgproc.Imgproc;
9+
import org.openqa.selenium.*;
10+
import org.openqa.selenium.interactions.Locatable;
11+
12+
import java.io.File;
13+
import java.util.ArrayList;
14+
import java.util.Comparator;
15+
import java.util.List;
16+
import java.util.stream.Collectors;
17+
18+
/**
19+
* Locator to search elements by image.
20+
* Takes screenshot and finds match using openCV.
21+
* Performs screenshot scaling if devicePixelRatio != 1.
22+
* Then finds elements by coordinates using javascript.
23+
*/
24+
public class ByImage extends By {
25+
private static boolean wasLibraryLoaded = false;
26+
private final Mat template;
27+
private final String description;
28+
private float threshold = 1 - AqualityServices.getConfiguration().getVisualizationConfiguration().getDefaultThreshold();
29+
30+
private static void loadLibrary() {
31+
if (!wasLibraryLoaded) {
32+
nu.pattern.OpenCV.loadShared();
33+
System.loadLibrary(org.opencv.core.Core.NATIVE_LIBRARY_NAME);
34+
wasLibraryLoaded = true;
35+
}
36+
}
37+
38+
/**
39+
* Constructor accepting image file.
40+
*
41+
* @param file image file to locate element by.
42+
*/
43+
public ByImage(File file) {
44+
loadLibrary();
45+
description = file.getName();
46+
this.template = Imgcodecs.imread(file.getAbsolutePath(), Imgcodecs.IMREAD_UNCHANGED);
47+
}
48+
49+
/**
50+
* Constructor accepting image bytes.
51+
*
52+
* @param bytes image bytes to locate element by.
53+
*/
54+
public ByImage(byte[] bytes) {
55+
loadLibrary();
56+
description = String.format("bytes[%d]", bytes.length);
57+
this.template = Imgcodecs.imdecode(new MatOfByte(bytes), Imgcodecs.IMREAD_UNCHANGED);
58+
}
59+
60+
/**
61+
* Sets threshold of image similarity.
62+
* @param threshold a float between 0 and 1, where 1 means 100% match, and 0.5 means 50% match.
63+
* @return current instance of ByImage locator.
64+
*/
65+
public ByImage setThreshold(float threshold) {
66+
if (threshold < 0 || threshold > 1) {
67+
throw new IllegalArgumentException("Threshold must be a float between 0 and 1.");
68+
}
69+
this.threshold = threshold;
70+
return this;
71+
}
72+
73+
/**
74+
* Gets threshold of image similarity.
75+
* @return current value of threshold.
76+
*/
77+
public float getThreshold() {
78+
return threshold;
79+
}
80+
81+
@Override
82+
public String toString() {
83+
return String.format("ByImage: %s, size: (width:%d, height:%d)", description, template.width(), template.height());
84+
}
85+
86+
@Override
87+
public boolean equals(Object o) {
88+
if (!(o instanceof ByImage)) {
89+
return false;
90+
}
91+
92+
ByImage that = (ByImage) o;
93+
94+
return this.template.equals(that.template);
95+
}
96+
97+
@Override
98+
public int hashCode() {
99+
return template.hashCode();
100+
}
101+
102+
103+
@Override
104+
public List<WebElement> findElements(SearchContext context) {
105+
Mat source = getScreenshot(context);
106+
Mat result = new Mat();
107+
Imgproc.matchTemplate(source, template, result, Imgproc.TM_CCOEFF_NORMED);
108+
109+
Core.MinMaxLocResult minMaxLoc = Core.minMaxLoc(result);
110+
111+
int matchCounter = Math.abs((result.width() - template.width() + 1) * (result.height() - template.height() + 1));
112+
List<Point> matchLocations = new ArrayList<>();
113+
while (matchCounter > 0 && minMaxLoc.maxVal >= threshold) {
114+
matchCounter--;
115+
Point matchLocation = minMaxLoc.maxLoc;
116+
matchLocations.add(matchLocation);
117+
Imgproc.rectangle(result, new Point(matchLocation.x, matchLocation.y), new Point(matchLocation.x + template.cols(),
118+
matchLocation.y + template.rows()), new Scalar(0, 0, 0), -1);
119+
minMaxLoc = Core.minMaxLoc(result);
120+
}
121+
122+
return matchLocations.stream().map(matchLocation -> getElementOnPoint(matchLocation, context)).collect(Collectors.toList());
123+
}
124+
125+
/**
126+
* Gets a single element on point (find by center coordinates, then select closest to matchLocation).
127+
*
128+
* @param matchLocation location of the upper-left point of the element.
129+
* @param context search context.
130+
* If the searchContext is Locatable (like WebElement), adjust coordinates to be absolute coordinates.
131+
* @return the closest found element.
132+
*/
133+
protected WebElement getElementOnPoint(Point matchLocation, SearchContext context) {
134+
if (context instanceof Locatable) {
135+
final org.openqa.selenium.Point point = ((Locatable) context).getCoordinates().onPage();
136+
matchLocation.x += point.getX();
137+
matchLocation.y += point.getY();
138+
}
139+
int centerX = (int) (matchLocation.x + (template.width() / 2));
140+
int centerY = (int) (matchLocation.y + (template.height() / 2));
141+
//noinspection unchecked
142+
List<WebElement> elements = (List<WebElement>) AqualityServices.getBrowser()
143+
.executeScript(JavaScript.GET_ELEMENTS_FROM_POINT, centerX, centerY);
144+
elements.sort(Comparator.comparingDouble(e -> distanceToPoint(matchLocation, e)));
145+
return elements.get(0);
146+
}
147+
148+
/**
149+
* Calculates distance from element to matching point.
150+
*
151+
* @param matchLocation matching point.
152+
* @param element target element.
153+
* @return distance in pixels.
154+
*/
155+
protected static double distanceToPoint(Point matchLocation, WebElement element) {
156+
org.openqa.selenium.Point elementLocation = element.getLocation();
157+
return Math.sqrt(Math.pow(matchLocation.x - elementLocation.x, 2) + Math.pow(matchLocation.y - elementLocation.y, 2));
158+
}
159+
160+
/**
161+
* Takes screenshot from searchContext if supported, or from browser.
162+
*
163+
* @param context search context for element location.
164+
* @return captured screenshot as Mat object.
165+
*/
166+
protected Mat getScreenshot(SearchContext context) {
167+
byte[] screenshotBytes = context instanceof TakesScreenshot
168+
? ((TakesScreenshot) context).getScreenshotAs(OutputType.BYTES)
169+
: AqualityServices.getBrowser().getScreenshot();
170+
boolean isBrowserScreenshot = context instanceof WebDriver || !(context instanceof TakesScreenshot);
171+
Mat source = Imgcodecs.imdecode(new MatOfByte(screenshotBytes), Imgcodecs.IMREAD_UNCHANGED);
172+
long devicePixelRatio = (long) AqualityServices.getBrowser().executeScript(JavaScript.GET_DEVICE_PIXEL_RATIO);
173+
if (devicePixelRatio != 1 && isBrowserScreenshot) {
174+
int scaledWidth = (int) (source.width() / devicePixelRatio);
175+
int scaledHeight = (int) (source.height() / devicePixelRatio);
176+
Imgproc.resize(source, source, new Size(scaledWidth, scaledHeight), 0, 0, Imgproc.INTER_AREA);
177+
}
178+
return source;
179+
}
180+
}

‎src/main/resources/js/getDevicePixelRatio.js

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
return window.devicePixelRatio;

‎src/main/resources/js/getElementsFromPoint.js

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
return document.elementsFromPoint(arguments[0], arguments[1]);

‎src/test/java/manytools/ManyToolsForm.java

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,13 @@
11
package manytools;
22

33
import aquality.selenium.browser.AqualityServices;
4+
import aquality.selenium.core.utilities.IActionRetrier;
45
import aquality.selenium.elements.interfaces.ILabel;
56
import aquality.selenium.forms.Form;
67
import org.openqa.selenium.By;
8+
import org.openqa.selenium.TimeoutException;
9+
10+
import java.util.Collections;
711

812
public abstract class ManyToolsForm<T extends ManyToolsForm<T>> extends Form {
913
private static final String BASE_URL = "https://manytools.org/";
@@ -17,8 +21,10 @@ protected ManyToolsForm(String name) {
1721

1822
@SuppressWarnings("unchecked")
1923
public T open() {
20-
AqualityServices.getBrowser().goTo(BASE_URL + getUrlPart());
21-
AqualityServices.getBrowser().waitForPageToLoad();
24+
AqualityServices.get(IActionRetrier.class).doWithRetry(() -> {
25+
AqualityServices.getBrowser().goTo(BASE_URL + getUrlPart());
26+
AqualityServices.getBrowser().waitForPageToLoad();
27+
}, Collections.singletonList(TimeoutException.class));
2228
return (T) this;
2329
}
2430

‎src/test/java/tests/integration/LocatorTests.java

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,23 @@
11
package tests.integration;
22

3+
import aquality.selenium.browser.AqualityServices;
4+
import aquality.selenium.elements.interfaces.ByImage;
35
import aquality.selenium.elements.interfaces.ILabel;
46
import automationpractice.forms.ChallengingDomForm;
57
import org.openqa.selenium.By;
8+
import org.openqa.selenium.OutputType;
69
import org.openqa.selenium.WebElement;
710
import org.openqa.selenium.support.locators.RelativeLocator;
11+
import org.testng.Assert;
812
import org.testng.annotations.BeforeMethod;
913
import org.testng.annotations.Test;
1014
import org.testng.asserts.SoftAssert;
1115
import tests.BaseTest;
1216
import theinternet.TheInternetPage;
17+
import theinternet.forms.BrokenImagesForm;
18+
1319
import java.util.List;
20+
1421
import static aquality.selenium.locators.RelativeBySupplier.with;
1522

1623
public class LocatorTests extends BaseTest {
@@ -26,6 +33,28 @@ public void beforeMethod() {
2633
navigate(TheInternetPage.CHALLENGING_DOM);
2734
}
2835

36+
@Test
37+
public void testByImageLocator() {
38+
BrokenImagesForm form = new BrokenImagesForm();
39+
Assert.assertFalse(form.getLabelByImage().state().isDisplayed(), "Should be impossible to find element on page by image when it is absent");
40+
getBrowser().goTo(form.getUrl());
41+
Assert.assertTrue(form.getLabelByImage().state().isDisplayed(), "Should be possible to find element on page by image");
42+
Assert.assertEquals(form.getLabelByImage().getElement().getTagName(), "img", "Correct element must be found");
43+
44+
List<ILabel> childLabels = form.getChildLabelsByImage();
45+
List<ILabel> docLabels = form.getLabelsByImage();
46+
Assert.assertTrue(docLabels.size() > 1, "List of elements should be possible to find by image");
47+
Assert.assertEquals(docLabels.size(), childLabels.size(), "Should be possible to find child elements by image with the same count");
48+
49+
ILabel documentByTag = AqualityServices.getElementFactory().getLabel(By.tagName("body"), "document by tag");
50+
float fullThreshold = 1;
51+
ILabel documentByImage = AqualityServices.getElementFactory().getLabel(new ByImage(documentByTag.getElement().getScreenshotAs(OutputType.BYTES)).setThreshold(fullThreshold),
52+
"body screen");
53+
Assert.assertTrue(documentByImage.state().isDisplayed(), "Should be possible to find element by document screenshot");
54+
Assert.assertEquals(((ByImage)documentByImage.getLocator()).getThreshold(), fullThreshold, "Should be possible to get ByImage threshold");
55+
Assert.assertEquals(documentByImage.getElement().getTagName(), "body", "Correct element must be found");
56+
}
57+
2958
@Test
3059
public void testAboveLocatorWithDifferentAboveParametersType() {
3160
ILabel cellInRow5Column5 = challengingDomForm.getCellInRow5Column5();

‎src/test/java/theinternet/forms/BrokenImagesForm.java

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
package theinternet.forms;
2+
3+
import aquality.selenium.elements.interfaces.ByImage;
4+
import aquality.selenium.elements.interfaces.ILabel;
5+
import org.openqa.selenium.By;
6+
import utils.FileUtil;
7+
8+
import java.util.List;
9+
10+
public class BrokenImagesForm extends TheInternetForm {
11+
private final By imageLocator = new ByImage(FileUtil.getResourceFileByName("brokenImage.png"));
12+
13+
public BrokenImagesForm(){
14+
super(By.id("content"), "Broken Images form");
15+
}
16+
17+
public ILabel getLabelByImage(){
18+
return getElementFactory().getLabel(imageLocator, "broken image");
19+
}
20+
21+
public List<ILabel> getLabelsByImage(){
22+
return getElementFactory().findElements(imageLocator, "broken image", ILabel.class);
23+
}
24+
25+
public List<ILabel> getChildLabelsByImage(){
26+
return getFormLabel().findChildElements(imageLocator, "broken image", ILabel.class);
27+
}
28+
29+
@Override
30+
protected String getUri() {
31+
return "/broken_images";
32+
}
33+
}

‎src/test/resources/brokenImage.png

765 Bytes
Loading

0 commit comments

Comments
 (0)