feat: Support multiple image match in findByImage (#449)

Author: khaong

lib/basedriver/commands/find.js

@@ -229,9 +229,7 @@ helpers.findByImage = async function findByImage (b64Template, {
       screenHeight);
   }
 
-  let rect = null;
-  let b64Matched = null;
-  let score = 0;
+  const results = [];
   const condition = async () => {
     try {
       const {b64Screenshot, scale} = await this.getScreenshotForImageFind(screenWidth, screenHeight);
@@ -241,12 +239,19 @@ helpers.findByImage = async function findByImage (b64Template, {
         fixImageTemplateScale, ...scale
       });
 
-      const comparedImage = await this.compareImages(MATCH_TEMPLATE_MODE, b64Screenshot, b64Template, {threshold, visualize});
-      rect = comparedImage.rect;
-      b64Matched = comparedImage.visualization;
-      score = comparedImage.score;
-
+      if (multiple) {
+        results.push(...(await this.compareImages(MATCH_TEMPLATE_MODE,
+                                                  b64Screenshot,
+                                                  b64Template,
+                                                  {threshold, visualize, multiple})));
+      } else {
+        results.push(await this.compareImages(MATCH_TEMPLATE_MODE,
+                                              b64Screenshot,
+                                              b64Template,
+                                              {threshold, visualize, multiple}));
+      }
       return true;
+
     } catch (err) {
       // if compareImages fails, we'll get a specific error, but we should
       // retry, so trap that and just return false to trigger the next round of
@@ -273,29 +278,28 @@ helpers.findByImage = async function findByImage (b64Template, {
     }
   }
 
-  if (!rect) {
+  if (_.isEmpty(results)) {
     if (multiple) {
       return [];
     }
     throw new errors.NoSuchElementError();
   }
 
-  log.info(`Image template matched: ${JSON.stringify(rect)}`);
-  if (b64Matched) {
-    log.info(`Matched base64 data: ${b64Matched.substring(0, 200)}...`);
-  }
-  const imgEl = new ImageElement(b64Template, rect, score, b64Matched);
+  const elements = results.map(({rect, score, visualization}) => {
+    log.info(`Image template matched: ${JSON.stringify(rect)}`);
+    return new ImageElement(b64Template, rect, score, visualization);
+  });
 
   // if we're just checking staleness, return straightaway so we don't add
   // a new element to the cache. shouldCheckStaleness does not support multiple
   // elements, since it is a purely internal mechanism
   if (shouldCheckStaleness) {
-    return imgEl;
+    return elements[0];
   }
 
-  const protocolEl = this.registerImageElement(imgEl);
+  const registeredElements = elements.map((imgEl) => this.registerImageElement(imgEl));
 
-  return multiple ? [protocolEl] : protocolEl;
+  return multiple ? registeredElements : registeredElements[0];
 };
 
 /**

lib/basedriver/commands/images.js

@@ -22,7 +22,21 @@ const DEFAULT_MATCH_THRESHOLD = 0.4;
  * @param {string} secondImage - Base64-encoded image file.
  * All image formats, that OpenCV library itself accepts, are supported.
  * @param {?Object} options [{}] - The content of this dictionary depends
- * on the actual `mode` value. See the documentation on `appium-support`
+ * on the actual `mode` value.
+ * For MATCH_TEMPLATE_MODE:
+ *   - visualize: include the visualization of the match in the result
+ *                (default: false)
+ *   - threshold: the image match threshold, higher values
+ *                require a closer image similarity to match
+ *                (default: 0.5)
+ *   - multiple: return multiple matches in the image
+ *               (default: false)
+ *   - matchNeighbourThreshold: if multiple is specified,
+ *                              determine the number of pixels within
+ *                              which to consider a pixel as part of the
+ *                              same match result
+ *                              (default: 10)
+ * See the documentation in the `appium-support`
  * module for more details.
  * @returns {Object} The content of the resulting dictionary depends
  * on the actual `mode` and `options` values. See the documentation on
@@ -34,7 +48,8 @@ const DEFAULT_MATCH_THRESHOLD = 0.4;
 commands.compareImages = async function compareImages (mode, firstImage, secondImage, options = {}) {
   const img1 = Buffer.from(firstImage, 'base64');
   const img2 = Buffer.from(secondImage, 'base64');
-  let result = {};
+  let result = null;
+
   switch (_.toLower(mode)) {
     case MATCH_FEATURES_MODE.toLowerCase():
       result = await imageUtil.getImagesMatches(img1, img2, options);
@@ -45,17 +60,34 @@ commands.compareImages = async function compareImages (mode, firstImage, secondI
     case MATCH_TEMPLATE_MODE.toLowerCase():
       // firstImage/img1 is the full image and secondImage/img2 is the partial one
       result = await imageUtil.getImageOccurrence(img1, img2, options);
+
+      if (options.multiple) {
+        return result.multiple.map(convertVisualizationToBase64);
+      }
       break;
     default:
       throw new errors.InvalidArgumentError(`'${mode}' images comparison mode is unknown. ` +
         `Only ${JSON.stringify([MATCH_FEATURES_MODE, GET_SIMILARITY_MODE, MATCH_TEMPLATE_MODE])} modes are supported.`);
   }
-  if (!_.isEmpty(result.visualization)) {
-    result.visualization = result.visualization.toString('base64');
-  }
-  return result;
+
+  return convertVisualizationToBase64(result);
 };
 
+/**
+ * base64 encodes the visualization part of the result
+ * (if necessary)
+ *
+ * @param {OccurenceResult} element - occurrence result
+ *
+ **/
+function convertVisualizationToBase64 (element) {
+  if (_.isString(element.visualization)) {
+    element.visualization = element.visualization.toString('base64');
+  }
+
+  return element;
+}
+
 Object.assign(extensions, commands, helpers);
 export { commands, helpers, DEFAULT_MATCH_THRESHOLD, MATCH_TEMPLATE_MODE };
 export default extensions;

package.json

@@ -35,7 +35,7 @@
   ],
   "dependencies": {
     "@babel/runtime": "^7.0.0",
-    "appium-support": "^2.48.0",
+    "appium-support": "^2.50.0",
     "async-lock": "^1.0.0",
     "asyncbox": "^2.3.1",
     "axios": "^0.x",

test/basedriver/commands/find-specs.js

@@ -74,7 +74,9 @@ describe('finding elements by image', function () {
     });
     it('should find image elements happypath', async function () {
       const d = new TestDriver();
-      basicStub(d);
+      const {compareStub} = basicStub(d);
+      compareStub.returns([{rect, score}]);
+
       const els = await d.findByImage(template, {multiple: true});
       els.should.have.length(1);
       basicImgElVerify(els[0], d);