From 39aa01db301eaac94e887c8c1cdc2f458ba6a077 Mon Sep 17 00:00:00 2001 From: Long Nguyen Date: Tue, 9 Nov 2021 19:34:25 +0900 Subject: [PATCH] fix: improve native web tap (strict) and support iOS 15 --- lib/commands/web.js | 346 ++++++++++++++++++++++---------------------- 1 file changed, 175 insertions(+), 171 deletions(-) diff --git a/lib/commands/web.js b/lib/commands/web.js index c6eb827b63..d697082f7b 100644 --- a/lib/commands/web.js +++ b/lib/commands/web.js @@ -1,5 +1,4 @@ -import { retryInterval } from 'asyncbox'; -import { util, timing } from '@appium/support'; +import { util, timing } from 'appium-support'; import log from '../logger'; import _ from 'lodash'; import B from 'bluebird'; @@ -7,22 +6,18 @@ import { errors, isErrorType } from '@appium/base-driver'; import cookieUtils from '../cookies'; import { EventEmitter } from 'events'; -const IPHONE_TOP_BAR_HEIGHT = 71; -const IPHONE_SCROLLED_TOP_BAR_HEIGHT = 41; -const IPHONE_X_SCROLLED_OFFSET = 55; -const IPHONE_X_NOTCH_OFFSET_IOS = 24; -const IPHONE_X_NOTCH_OFFSET_IOS_13 = 20; - -const IPHONE_LANDSCAPE_TOP_BAR_HEIGHT = 51; -const IPHONE_BOTTOM_BAR_OFFSET = 49; -const TAB_BAR_OFFSET = 33; -const IPHONE_WEB_COORD_SMART_APP_BANNER_OFFSET = 84; -const IPAD_WEB_COORD_SMART_APP_BANNER_OFFSET = 95; +const PORTRAIT = 'portrait'; +const LANDSCAPE = 'landscape'; +// Resolution Ref: https://www.ios-resolution.com/ +// Need to update when Apple releases new notch models const NOTCHED_DEVICE_SIZES = [ {w: 1125, h: 2436}, // 11 Pro, X, Xs {w: 828, h: 1792}, // 11, Xr {w: 1242, h: 2688}, // 11 Pro Max, Xs Max + {w: 1080, h: 2340}, // 12 mini, 13 mini + {w: 1170, h: 2532}, // 12, 12 Pro, 13, 13 Pro + {w: 1284, h: 2778}, // 12 Pro Max, 13 Pro Max ]; const { W3C_WEB_ELEMENT_IDENTIFIER } = util; @@ -32,16 +27,11 @@ const ATOM_INITIAL_WAIT_MS = 1000; const ON_OBSTRUCTING_ALERT_EVENT = 'alert'; -const VISIBLE = 'visible'; -const INVISIBLE = 'invisible'; -const DETECT = 'detect'; -const VISIBILITIES = [VISIBLE, INVISIBLE, DETECT]; - // The position of Safari's tab (search bar). // Since iOS 15, the bar is the bottom by default. const TAB_BAR_POSITION_TOP = 'top'; const TAB_BAR_POSITION_BOTTOM = 'bottom'; -const TAB_BAR_POSSITIONS = [TAB_BAR_POSITION_TOP, TAB_BAR_POSITION_BOTTOM]; +const TAB_BAR_POSITIONS = [TAB_BAR_POSITION_TOP, TAB_BAR_POSITION_BOTTOM]; const commands = {}, helpers = {}, extensions = {}; @@ -333,21 +323,57 @@ extensions.getSafariIsIphone = _.memoize(async function getSafariIsIphone () { return true; }); -extensions.getSafariDeviceSize = _.memoize(async function getSafariDeviceSize () { - const script = 'return {height: window.screen.availHeight * window.devicePixelRatio, width: window.screen.availWidth * window.devicePixelRatio};'; - const {width, height} = await this.execute(script); - const [normHeight, normWidth] = height > width ? [height, width] : [width, height]; - return { - width: normWidth, - height: normHeight, - }; +extensions.getSafariIsIpad = _.memoize(async function getSafariIsIpad () { + try { + return await this.execute(`return (navigator.platform === 'MacIntel' && navigator.maxTouchPoints > 0) || navigator.platform === 'iPad'`); + } catch (ign) { + return false; + } }); +extensions.getSafariDeviceSize = async function getSafariDeviceSize () { + // Why did I not use availWidth/Height? + // It seems that Safari on iPad is not able to get the correct size of the screen on different orientations + const getSizeScript = + `let { innerWidth, innerHeight, screen, devicePixelRatio } = window; + const { height, width } = screen; + const physicalWidth = width * devicePixelRatio; + const physicalHeight = height * devicePixelRatio; + const orientation = height > width ? '${PORTRAIT}' : '${LANDSCAPE}'; + let scaleUpRatio = 1;`; + + // - Viewport gave wrong value of innerHeight/innerWidth in some websites on iPad. (Ed: google.com) + // Ref: https://stackoverflow.com/questions/4629969/ios-return-bad-value-for-window-innerheight-width + // Though the innerHeight/innerWidth is not correct, but the ratio of physicalWidth/physicalHeight is correct. + // - The scale-up ratio is used to convert the coordinates got from API to correct coordinates.. + const fixIPadIssueScript = await this.getSafariIsIpad() ? + `// Assuming the viewport width is matching the device width + const innerRatio = innerHeight / innerWidth; + scaleUpRatio = innerWidth / width; + // Reset innerWidth/innerHeight + innerWidth = width; + innerHeight = Math.round(width * innerRatio);` : ''; + + const returnScript = + `return { + screenWidth: width, + screenHeight: height, + webviewWidth: innerWidth, + webviewHeight: innerHeight, + physicalWidth, + physicalHeight, + orientation, + scaleUpRatio, + };`; + + return await this.execute(getSizeScript + fixIPadIssueScript + returnScript); +}; + extensions.getSafariIsNotched = _.memoize(async function getSafariIsNotched () { try { - const {width, height} = await this.getSafariDeviceSize(); + const {physicalWidth, physicalHeight} = await this.getSafariDeviceSize(); for (const device of NOTCHED_DEVICE_SIZES) { - if (device.w === width && device.h === height) { + if (device.w === physicalWidth && device.h === physicalHeight) { return true; } } @@ -358,114 +384,96 @@ extensions.getSafariIsNotched = _.memoize(async function getSafariIsNotched () { return false; }); -extensions.getExtraTranslateWebCoordsOffset = async function getExtraTranslateWebCoordsOffset (wvPos, realDims) { - let topOffset = 0; - let bottomOffset = 0; - - const isIphone = await this.getSafariIsIphone(); - - // No need to check whether the Smart App Banner or Tab Bar is visible or not - // if already defined by nativeWebTapTabBarVisibility or nativeWebTapSmartAppBannerVisibility in settings. - const { - nativeWebTapTabBarVisibility, - nativeWebTapSmartAppBannerVisibility, - safariTabBarPosition = util.compareVersions(this.opts.platformVersion, '>=', '15.0') && isIphone - ? TAB_BAR_POSITION_BOTTOM : TAB_BAR_POSITION_TOP, - } = await this.settings.getSettings(); - let tabBarVisibility = _.lowerCase(nativeWebTapTabBarVisibility); - let bannerVisibility = _.lowerCase(nativeWebTapSmartAppBannerVisibility); - const tabBarPosition = _.lowerCase(safariTabBarPosition); - - if (!VISIBILITIES.includes(tabBarVisibility)) { - tabBarVisibility = DETECT; - } - if (!VISIBILITIES.includes(bannerVisibility)) { - bannerVisibility = DETECT; +extensions.isToolbarCollapsed = _.memoize(function isToolbarCollapsed (isIphone, orientation, screenHeight, webviewHeight) { + const remainHeight = screenHeight - webviewHeight; + if (isIphone) { + if (orientation === PORTRAIT) { + return remainHeight < 100; + } else if (orientation === LANDSCAPE) { + return remainHeight === 0; + } + } else { + return remainHeight < 50; } +}, (...args) => JSON.stringify(args)); - if (!TAB_BAR_POSSITIONS.includes(tabBarPosition)) { +extensions.getWebViewRect = async function getWebViewRect () { + const isIphone = await this.getSafariIsIphone(); + const isNotched = isIphone && await this.getSafariIsNotched(); + const {screenWidth, screenHeight, webviewWidth, webviewHeight, orientation, scaleUpRatio} = await this.getSafariDeviceSize(); + const remainHeight = screenHeight - webviewHeight; + const remainWidth = screenWidth - webviewWidth; + const isCollapsed = this.isToolbarCollapsed(isIphone, orientation, screenHeight, webviewHeight); + + const { safariTabBarPosition = TAB_BAR_POSITION_BOTTOM } = await this.settings.getSettings(); + + if ( + isIphone && + util.compareVersions(this.opts.platformVersion, '>=', '15.0') && + !TAB_BAR_POSITIONS.includes(safariTabBarPosition) + ) { throw new errors.InvalidArgumentError( - `${safariTabBarPosition} is invalid as Safari tab bar position. Available positions are ${TAB_BAR_POSSITIONS}.`); + `${safariTabBarPosition} is invalid as Safari tab bar position. Available positions are ${TAB_BAR_POSITIONS}.` + ); } - const isNotched = isIphone && await this.getSafariIsNotched(); - - const orientation = realDims.h > realDims.w ? 'PORTRAIT' : 'LANDSCAPE'; - - const notchOffset = isNotched - ? util.compareVersions(this.opts.platformVersion, '=', '13.0') - ? IPHONE_X_NOTCH_OFFSET_IOS_13 - : IPHONE_X_NOTCH_OFFSET_IOS - : 0; - - const isScrolled = await this.execute('return document.documentElement.scrollTop > 0'); - if (isScrolled) { - topOffset = IPHONE_SCROLLED_TOP_BAR_HEIGHT + notchOffset; - - if (isNotched) { - topOffset -= IPHONE_X_SCROLLED_OFFSET; - } - - // If the iPhone is landscape then there is no top bar - if (orientation === 'LANDSCAPE' && isIphone) { - topOffset = 0; + let x = 0; + let y = 0; + + if ( + ((util.compareVersions(this.opts.platformVersion, '>=', '15.0') && + safariTabBarPosition === TAB_BAR_POSITION_TOP) || + util.compareVersions(this.opts.platformVersion, '<', '15.0')) && + isIphone && + orientation === PORTRAIT + ) { + if (isCollapsed) { + // offset with actual: 1 + y = remainHeight + 1; + } else if (isNotched) { + // offset with actual: -3, fixed bottom: 49, fixed indicator: 30 + y = remainHeight - 82; // = - 3 - 49 - 30 + } else { + // offset with actual: 1, fixed bottom: 44 + y = remainHeight - 43; // = 1 - 44 } - } else { - topOffset = tabBarPosition === TAB_BAR_POSITION_BOTTOM ? 0 : IPHONE_TOP_BAR_HEIGHT; - topOffset += notchOffset; - log.debug(`tabBarPosition and topOffset: ${tabBarPosition}, ${topOffset}`); - + } else if (util.compareVersions(this.opts.platformVersion, '<', '15.0')) { if (isIphone) { - if (orientation === 'PORTRAIT') { - // The bottom bar is only visible when portrait - bottomOffset = IPHONE_BOTTOM_BAR_OFFSET; - } else { - topOffset = IPHONE_LANDSCAPE_TOP_BAR_HEIGHT; - } - } - - if (orientation === 'LANDSCAPE' || !isIphone) { - if (tabBarVisibility === VISIBLE) { - topOffset += TAB_BAR_OFFSET; - } else if (tabBarVisibility === DETECT) { - // Tabs only appear if the device is landscape or if it's an iPad so we only check visibility in this case - // Assume that each tab bar is a WebView - const contextsAndViews = await this.getContextsAndViews(); - const tabs = contextsAndViews.filter((ctx) => ctx.id.startsWith('WEBVIEW_')); - - if (tabs.length > 1) { - log.debug(`Found ${tabs.length} tabs. Assuming the tab bar is visible`); - topOffset += TAB_BAR_OFFSET; + if (isNotched) { + x = remainWidth / 2; + if (!isCollapsed) { + // offset with actual: 1 + y = remainHeight + 1; } } + } else { + // offset with actual: 1 + y = remainHeight + 1; } - } - - topOffset += await this.getExtraNativeWebTapOffset(isIphone, bannerVisibility); - - wvPos.y += topOffset; - realDims.h -= (topOffset + bottomOffset); -}; - -extensions.getExtraNativeWebTapOffset = async function getExtraNativeWebTapOffset (isIphone, bannerVisibility) { - let offset = 0; - - if (bannerVisibility === VISIBLE) { - offset += isIphone ? - IPHONE_WEB_COORD_SMART_APP_BANNER_OFFSET : - IPAD_WEB_COORD_SMART_APP_BANNER_OFFSET; - } else if (bannerVisibility === DETECT) { - // try to see if there is an Smart App Banner - const banners = await this.findNativeElementOrElements('accessibility id', 'Close app download offer', true); - if (banners.length > 0) { - offset += isIphone ? - IPHONE_WEB_COORD_SMART_APP_BANNER_OFFSET : - IPAD_WEB_COORD_SMART_APP_BANNER_OFFSET; + } else if (isIphone) { + if (orientation === PORTRAIT) { + if (isNotched) { + // Fixed top: 47 + y = 47; + } else { + // Fixed status bar: 20 + y = 20; + } + } else if (isNotched) { + x = remainWidth / 2; + if (!isCollapsed) { + // Fixed top: 48 + y = 48; + } + } else if (!isCollapsed) { + // Fixed top: 60 + y = 60; } + } else { + y = remainHeight; } - log.debug(`Additional native web tap offset computed: ${offset}`); - return offset; + return {x, y, webviewWidth, webviewHeight, scaleUpRatio, screenWidth, screenHeight, orientation}; }; async function tapWebElementNatively (driver, atomsElement) { @@ -529,6 +537,7 @@ extensions.nativeWebTap = async function nativeWebTap (el) { this.executeAtom('get_size', [atomsElement]), this.executeAtom('get_top_left_coordinates', [atomsElement]), ]); + const {width, height} = size; let {x, y} = coordinates; x += width / 2; @@ -539,68 +548,63 @@ extensions.nativeWebTap = async function nativeWebTap (el) { }; extensions.clickCoords = async function clickCoords (coords) { - await this.performTouch([ + await this.performActions([ { - action: 'tap', - options: coords, - }, + type: 'pointer', + id: 'finger1', + parameters: { + pointerType: 'touch' + }, + actions: [ + { + type: 'pointerMove', + duration: 0, + x: coords.x, + y: coords.y, + }, + { + type: 'pointerDown', + button: 0 + }, + { + type: 'pointerUp', + button: 0 + } + ] + } ]); }; extensions.translateWebCoords = async function translateWebCoords (coords) { - log.debug(`Translating coordinates (${JSON.stringify(coords)}) to web coordinates`); - - // absolutize web coords - let webview; - try { - webview = await retryInterval(5, 100, async () => await this.findNativeElementOrElements('class name', 'XCUIElementTypeWebView', false)); - } catch (ign) {} - - if (!webview) { - throw new Error(`No WebView found. Unable to translate web coordinates for native web tap.`); - } - - webview = util.unwrapElement(webview); - - const rect = await this.proxyCommand(`/element/${webview}/rect`, 'GET'); - const wvPos = {x: rect.x, y: rect.y}; - const realDims = {w: rect.width, h: rect.height}; - - const cmd = '(function () { return {w: window.innerWidth, h: window.innerHeight}; })()'; - const wvDims = await this.remote.execute(cmd); + log.debug(`Translating web coordinates (${JSON.stringify(coords)}) to real coordinates`); // keep track of implicit wait, and set locally to 0 // https://github.com/appium/appium/issues/14988 const implicitWaitMs = this.implicitWaitMs; await this.setImplicitWait(0); + let wvRect = {x: 0, y: 0}; try { - await this.getExtraTranslateWebCoordsOffset(wvPos, realDims); + wvRect = await this.getWebViewRect(); } finally { await this.setImplicitWait(implicitWaitMs); } + const {x, y, webviewWidth, webviewHeight, scaleUpRatio, screenWidth, screenHeight, orientation} = wvRect; + const newCoords = { + // x, y are already scaled, need to get real coordinates. + x: Math.round(coords.x / scaleUpRatio) + x, + y: Math.round(coords.y / scaleUpRatio) + y, + }; - if (wvDims && realDims && wvPos) { - let xRatio = realDims.w / wvDims.w; - let yRatio = realDims.h / wvDims.h; - let newCoords = { - x: wvPos.x + Math.round(xRatio * coords.x), - y: wvPos.y + Math.round(yRatio * coords.y), - }; - - // additional logging for coordinates, since it is sometimes broken - // see https://github.com/appium/appium/issues/9159 - log.debug(`Converted coordinates: ${JSON.stringify(newCoords)}`); - log.debug(` rect: ${JSON.stringify(rect)}`); - log.debug(` wvPos: ${JSON.stringify(wvPos)}`); - log.debug(` realDims: ${JSON.stringify(realDims)}`); - log.debug(` wvDims: ${JSON.stringify(wvDims)}`); - log.debug(` xRatio: ${JSON.stringify(xRatio)}`); - log.debug(` yRatio: ${JSON.stringify(yRatio)}`); - - log.debug(`Converted web coords ${JSON.stringify(coords)} ` + - `into real coords ${JSON.stringify(newCoords)}`); - return newCoords; - } + // additional logging for coordinates, since it is sometimes broken + // see https://github.com/appium/appium/issues/9159 + log.debug(`Converted web coords ${JSON.stringify(coords)} ` + + `into real coords ${JSON.stringify(newCoords)}`); + log.debug(` Webview coords: ${JSON.stringify({x, y})}`); + log.debug(` Webview dimension: ${JSON.stringify({w: webviewWidth, h: webviewHeight})}`); + log.debug(` Screen dimension: ${JSON.stringify({w: screenWidth, h: screenHeight})}`); + log.debug(` Orientation: ${orientation}`); + log.debug(` Scale ratio: ${scaleUpRatio}`); + return newCoords; }; extensions.checkForAlert = async function checkForAlert () {