Add home and Siri button for iOS devices

README.md

@@ -47,7 +47,9 @@ any other command: `./Appium-Inspector-linux.AppImage`.
 * Get a list of suggested element locator strategies and selectors to be used in your scripts
 * Compare the speed of different element finding strategies
 * Start and stop "recording" mode, which translates your actions in the Inspector to code samples you can use in your scripts
+* Start and stop "source refreshing", which allows you to interact with screen without reloading page source after every request
 * Tap on the screen at an arbitrary location
+* Press home button for iOS and Android
 * Perform a swipe gesture
 * Switch into web context modes and interact with web elements
 * Test out your own locator strategies

app/renderer/actions/Inspector.js

@@ -46,6 +46,9 @@ export const SET_SHOW_BOILERPLATE = 'SET_SHOW_BOILERPLATE';
 
 export const SHOW_LOCATOR_TEST_MODAL = 'SHOW_LOCATOR_TEST_MODAL';
 export const HIDE_LOCATOR_TEST_MODAL = 'HIDE_LOCATOR_TEST_MODAL';
+export const SHOW_SIRI_COMMAND_MODAL = 'SHOW_SIRI_COMMAND_MODAL';
+export const HIDE_SIRI_COMMAND_MODAL = 'HIDE_SIRI_COMMAND_MODAL';
+export const SET_SIRI_COMMAND_VALUE = 'SET_SIRI_COMMAND_VALUE';
 export const SET_LOCATOR_TEST_STRATEGY = 'SET_LOCATOR_TEST_STRATEGY';
 export const SET_LOCATOR_TEST_VALUE = 'SET_LOCATOR_TEST_VALUE';
 export const SEARCHING_FOR_ELEMENTS = 'SEARCHING_FOR_ELEMENTS';
@@ -106,8 +109,9 @@ export const SET_GESTURE_TAP_COORDS_MODE = 'SET_GESTURE_TAP_COORDS_MODE';
 export const CLEAR_TAP_COORDINATES = 'CLEAR_TAP_COORDINATES';
 
 export const TOGGLE_SHOW_ATTRIBUTES = 'TOGGLE_SHOW_ATTRIBUTES';
+export const TOGGLE_REFRESHING_STATE = 'TOGGLE_REFRESHING_STATE';
 
-const KEEP_ALIVE_PING_INTERVAL = 20 * 1000;
+const KEEP_ALIVE_PING_INTERVAL = 5 * 1000;
 const NO_NEW_COMMAND_LIMIT = 24 * 60 * 60 * 1000; // Set timeout to 24 hours
 const WAIT_FOR_USER_KEEP_ALIVE = 60 * 60 * 1000; // Give user 1 hour to reply
 
@@ -391,6 +395,25 @@ export function hideLocatorTestModal () {
   };
 }
 
+export function showSiriCommandModal () {
+  return (dispatch) => {
+    dispatch({type: SHOW_SIRI_COMMAND_MODAL});
+  };
+}
+
+export function hideSiriCommandModal () {
+  return (dispatch) => {
+    dispatch({type: HIDE_SIRI_COMMAND_MODAL});
+    dispatch({type: CLEAR_SEARCHED_FOR_ELEMENT_BOUNDS});
+  };
+}
+
+export function setSiriCommandValue (siriCommandValue) {
+  return (dispatch) => {
+    dispatch({type: SET_SIRI_COMMAND_VALUE, siriCommandValue});
+  };
+}
+
 export function setLocatorTestValue (locatorTestValue) {
   return (dispatch) => {
     dispatch({type: SET_LOCATOR_TEST_VALUE, locatorTestValue});
@@ -500,6 +523,12 @@ export function selectScreenshotInteractionMode (screenshotInteractionMode) {
   };
 }
 
+export function toggleRefreshingState () {
+  return (dispatch) => {
+    dispatch({type: TOGGLE_REFRESHING_STATE});
+  };
+}
+
 export function selectAppMode (mode) {
   return async (dispatch, getState) => {
     const {appMode} = getState().inspector;
@@ -692,7 +721,7 @@ export function keepSessionAlive () {
 
 export function callClientMethod (params) {
   return async (dispatch, getState) => {
-    const {driver, appMode, mjpegScreenshotUrl} = getState().inspector;
+    const {driver, appMode, mjpegScreenshotUrl, isRefreshingSource} = getState().inspector;
     const {methodName, ignoreResult = true} = params;
     params.appMode = appMode;
 
@@ -701,6 +730,10 @@ export function callClientMethod (params) {
       params.skipScreenshot = true;
     }
 
+    if (!isRefreshingSource) {
+      params.skipRefresh = true;
+    }
+
     console.log(`Calling client method with params:`); // eslint-disable-line no-console
     console.log(params); // eslint-disable-line no-console
     const action = keepSessionAlive();

app/renderer/actions/Session.js

@@ -74,6 +74,7 @@ const FILE_PATH_STORAGE_KEY = 'last_opened_file';
 const AUTO_START_URL_PARAM = '1'; // what should be passed in to ?autoStart= to turn it on
 
 const MJPEG_CAP = 'mjpegScreenshotUrl';
+const MJPEG_PORT_CAP = 'mjpegServerPort';
 
 // Multiple requests sometimes send a new session request
 // after establishing a session.
@@ -552,10 +553,18 @@ export function newSession (caps, attachSessId = null) {
     }
 
 
-    const mjpegScreenshotUrl = desiredCapabilities[`appium:${MJPEG_CAP}`] ||
-      desiredCapabilities[MJPEG_CAP] ||
+    var mjpegScreenshotUrl = driver.capabilities[`appium:${MJPEG_CAP}`] ||
+      driver.capabilities[MJPEG_CAP] ||
+      null;
+
+    const mjpegScreenshotPort = driver.capabilities[`appium:${MJPEG_PORT_CAP}`] ||
+      driver.capabilities[MJPEG_PORT_CAP] ||
       null;
 
+    // Build mjpegScreenshotUrl if mjpegServerPort in session capabilities
+    if (!mjpegScreenshotUrl && mjpegScreenshotPort) {
+      mjpegScreenshotUrl = `${https ? 'https' : 'http'}://${host}:${mjpegScreenshotPort}`;
+    }
 
     // pass some state to the inspector that it needs to build recorder
     // code boilerplate
@@ -571,7 +580,7 @@ export function newSession (caps, attachSessId = null) {
         https,
       },
       mode,
-      mjpegScreenshotUrl,
+      mjpegScreenshotUrl
     });
     action(dispatch);
     dispatch(push('/inspector'));

app/renderer/components/Inspector/HighlighterRects.js

@@ -225,7 +225,7 @@ export default class HighlighterRects extends Component {
 
   render () {
     const {source, screenshotInteractionMode, containerEl, searchedForElementBounds,
-           isLocatorTestModalVisible, scaleRatio, showCentroids} = this.props;
+           isLocatorTestModalVisible, isSiriCommandModalVisible, scaleRatio, showCentroids} = this.props;
 
     // Array of all element objects with properties to draw rectangles and/or centroids
     const elements = this.getElements(source);
@@ -279,7 +279,7 @@ export default class HighlighterRects extends Component {
     }
 
     // Don't show highlighter rects when Search Elements modal is open
-    if (!isLocatorTestModalVisible) {
+    if (!isLocatorTestModalVisible && !isSiriCommandModalVisible) {
       renderElements(elements);
       if (showCentroids) {
         renderCentroids(elements);

app/renderer/components/Inspector/Inspector.js

@@ -20,6 +20,8 @@ import {
   CheckCircleOutlined,
   CloseCircleOutlined,
   ReloadOutlined,
+  PlayCircleOutlined,
+  PauseCircleOutlined,
   EyeOutlined,
   PauseOutlined,
   SearchOutlined,
@@ -34,6 +36,8 @@ import {
   AppstoreOutlined,
   GlobalOutlined,
   CodeOutlined,
+  HomeOutlined,
+  SoundOutlined
 } from '@ant-design/icons';
 import { BUTTON } from '../../../../gui-common/components/AntdTypes';
 
@@ -188,18 +192,43 @@ export default class Inspector extends Component {
   render () {
     const {screenshot, screenshotError, selectedElement = {},
            applyClientMethod, quitSession, isRecording, showRecord, startRecording,
-           pauseRecording, showLocatorTestModal, appMode,
+           pauseRecording, showLocatorTestModal, showSiriCommandModal, appMode,
            screenshotInteractionMode, isFindingElementsTimes, visibleCommandMethod,
            selectedInteractionMode, selectInteractionMode, selectAppMode, setVisibleCommandResult,
            showKeepAlivePrompt, keepSessionAlive, sourceXML, t, visibleCommandResult,
            mjpegScreenshotUrl, isAwaitingMjpegStream, toggleShowCentroids, showCentroids,
-           isGestureEditorVisible, toggleShowAttributes} = this.props;
+           isGestureEditorVisible, toggleShowAttributes, isRefreshingSource, toggleRefreshingState, driver
+           } = this.props;
     const {path} = selectedElement;
 
     const showScreenshot = ((screenshot && !screenshotError) ||
-                            (mjpegScreenshotUrl && !isAwaitingMjpegStream));
+                            (mjpegScreenshotUrl && (!isRefreshingSource || !isAwaitingMjpegStream)));
 
     let screenShotControls = <div className={InspectorStyles['screenshot-controls']}>
+      {driver.client.isIOS &&
+      <div className={InspectorStyles['action-controls']}>
+        <Tooltip title={t('Press Home button')}>
+          <Button id='btnPressHomeButton' icon={<HomeOutlined/>} onClick={() => applyClientMethod({ methodName: 'executeScript', args: ['mobile:pressButton', [{name: 'home'}]]})}/>
+        </Tooltip>
+        <Tooltip title={t('Siri command')}>
+          <Button id='siriCommand' icon={<SoundOutlined/>} onClick={showSiriCommandModal} />
+        </Tooltip>
+      </div>
+      }
+      {/* {driver.client.isTVOS &&
+      <div className={InspectorStyles['action-controls']}>
+        <Tooltip title={t('Press Home button')}>
+          <Button id='btnPressHomeButton' icon={<HomeOutlined/>} onClick={() => applyClientMethod({ methodName: 'executeScript', args: ['mobile:pressButton', [{name: 'home'}]]})}/>
+        </Tooltip>
+      </div>
+      } */}
+      {driver.client.isAndroid &&
+      <div className={InspectorStyles['action-controls']}>
+        <Tooltip title={t('Press Home button')}>
+          <Button id='btnPressHomeButton' icon={<HomeOutlined/>} onClick={() => applyClientMethod({ methodName: 'pressKeycode', args: [{keycode: 10}]})}/>
+        </Tooltip>
+      </div>
+      }
       <div className={InspectorStyles['action-controls']}>
         <Tooltip title={t(showCentroids ? 'Hide Element Handles' : 'Show Element Handles')} placement="topRight">
           <Switch
@@ -335,6 +364,16 @@ export default class Inspector extends Component {
       <Tooltip title={t('Back')}>
         <Button id='btnGoBack' icon={<ArrowLeftOutlined/>} onClick={() => applyClientMethod({methodName: 'back'})}/>
       </Tooltip>
+      {!isRefreshingSource &&
+        <Tooltip title={t('Start Refreshing Source')}>
+          <Button id='btnStartRefreshing' icon={<PlayCircleOutlined/>} onClick={toggleRefreshingState}/>
+        </Tooltip>
+      }
+      {isRefreshingSource &&
+        <Tooltip title={t('Pause Refreshing Source')}>
+          <Button id='btnPauseRefreshing' icon={<PauseCircleOutlined/>} onClick={toggleRefreshingState}/>
+        </Tooltip>
+      }
       <Tooltip title={t('refreshSource')}>
         <Button id='btnReload' icon={<ReloadOutlined/>} onClick={() => applyClientMethod({methodName: 'getPageSource'})}/>
       </Tooltip>

app/renderer/components/Inspector/Screenshot.js

@@ -147,9 +147,12 @@ class Screenshot extends Component {
       scaleRatio,
       selectedTick,
       selectedInteractionMode,
+      isRefreshingSource
     } = this.props;
     const {x, y} = this.state;
 
+    const showSpinner = (isRefreshingSource && !!methodCallInProgress);
+
     // If we're tapping or swiping, show the 'crosshair' cursor style
     const screenshotStyle = {};
     if ([TAP, SWIPE].includes(screenshotInteractionMode) || selectedTick) {
@@ -171,7 +174,7 @@ class Screenshot extends Component {
     const points = this.getGestureCoordinates();
 
     // Show the screenshot and highlighter rects. Show loading indicator if a method call is in progress.
-    return <Spin size='large' spinning={!!methodCallInProgress}>
+    return <Spin size='large' spinning={showSpinner}>
       <div className={styles.innerScreenshotContainer}>
         <div ref={(containerEl) => { this.containerEl = containerEl; }}
           style={screenshotStyle}

app/renderer/components/Inspector/SiriCommandModal.js

@@ -0,0 +1,48 @@
+import React, { Component } from 'react';
+import { Modal, Button, Input, Row  } from 'antd';
+import { hideSiriCommandModal, setSiriCommandValue } from '../../actions/Inspector';
+import InspectorStyles from './Inspector.css';
+import { withTranslation } from '../../util';
+
+class SiriCommandModal extends Component {
+
+  onSubmit () {
+    const {
+      siriCommandValue,
+      hideSiriCommandModal,
+      applyClientMethod
+    } = this.props;
+    applyClientMethod({ methodName: 'executeScript', args: ['mobile:siriCommand', [{text: siriCommandValue}]]});
+    hideSiriCommandModal();
+  }
+
+  onCancel () {
+    const {hideSiriCommandModal} = this.props;
+    hideSiriCommandModal();
+  }
+
+  render () {
+    const {
+      siriCommandValue,
+      setSiriCommandValue,
+      isSiriCommandModalVisible,
+      t,
+    } = this.props;
+
+    // Footer displays all the buttons at the bottom of the Modal
+    return <Modal visible={isSiriCommandModalVisible}
+      title={t('Execute Siri Command')}
+      onCancel={this.onCancel.bind(this)}
+      footer=
+        {[
+          <Button onClick={this.onSubmit.bind(this)} type="primary">Execute command</Button>
+        ]}>
+      <Row>
+        {t('Command')}
+        <Input.TextArea onChange={(e) => setSiriCommandValue(e.target.value)} value={siriCommandValue} />
+      </Row>
+    </Modal>;
+  }
+}
+
+export default withTranslation(SiriCommandModal);

app/renderer/components/Inspector/Source.js

@@ -3,6 +3,7 @@ import { Tree } from 'antd';
 import LocatorTestModal from './LocatorTestModal';
 import InspectorStyles from './Inspector.css';
 import { withTranslation } from '../../util';
+import SiriCommandModal from './SiriCommandModal';
 
 const IMPORTANT_ATTRS = [
   'name',
@@ -97,6 +98,7 @@ class Source extends Component {
         sourceError && t('couldNotObtainSource', {errorMsg: JSON.stringify(sourceError)})
       }
       <LocatorTestModal {...this.props} />
+      <SiriCommandModal {...this.props} />
     </div>;
   }
 }

app/renderer/lib/appium-client.js

@@ -105,11 +105,11 @@ export default class AppiumClient {
       }
     }
 
-    // Give the source/screenshot time to change before taking the screenshot
-    await Bluebird.delay(500);
 
     let contextUpdate = {}, sourceUpdate = {}, screenshotUpdate = {}, windowSizeUpdate = {};
     if (!skipRefresh) {
+      // Give the source/screenshot time to change before taking the screenshot
+      await Bluebird.delay(500);
       if (!skipScreenshot) {
         screenshotUpdate = await this.getScreenshotUpdate();
       }

app/renderer/reducers/Inspector.js

@@ -5,7 +5,7 @@ import { SET_SOURCE_AND_SCREENSHOT, QUIT_SESSION_REQUESTED, QUIT_SESSION_DONE,
          SET_FIELD_VALUE, SET_EXPANDED_PATHS, SHOW_SEND_KEYS_MODAL,
          HIDE_SEND_KEYS_MODAL, START_RECORDING, PAUSE_RECORDING, CLEAR_RECORDING,
          SET_ACTION_FRAMEWORK, RECORD_ACTION, CLOSE_RECORDER, SET_SHOW_BOILERPLATE, SET_SESSION_DETAILS,
-         SHOW_LOCATOR_TEST_MODAL, HIDE_LOCATOR_TEST_MODAL, SET_LOCATOR_TEST_STRATEGY, SET_LOCATOR_TEST_VALUE,
+         SHOW_LOCATOR_TEST_MODAL, HIDE_LOCATOR_TEST_MODAL, SHOW_SIRI_COMMAND_MODAL, HIDE_SIRI_COMMAND_MODAL, SET_LOCATOR_TEST_STRATEGY, SET_LOCATOR_TEST_VALUE,
          SEARCHING_FOR_ELEMENTS, SEARCHING_FOR_ELEMENTS_COMPLETED, SET_LOCATOR_TEST_ELEMENT, CLEAR_SEARCH_RESULTS,
          ADD_ASSIGNED_VAR_CACHE, CLEAR_ASSIGNED_VAR_CACHE, SET_SCREENSHOT_INTERACTION_MODE,
          SET_SWIPE_START, SET_SWIPE_END, CLEAR_SWIPE_ACTION, SET_SEARCHED_FOR_ELEMENT_BOUNDS, CLEAR_SEARCHED_FOR_ELEMENT_BOUNDS,
@@ -17,7 +17,7 @@ import { SET_SOURCE_AND_SCREENSHOT, QUIT_SESSION_REQUESTED, QUIT_SESSION_DONE,
          GET_SAVED_GESTURES_REQUESTED, GET_SAVED_GESTURES_DONE, SET_LOADED_GESTURE, REMOVE_LOADED_GESTURE, SHOW_GESTURE_ACTION, HIDE_GESTURE_ACTION,
          SELECT_TICK_ELEMENT, UNSELECT_TICK_ELEMENT, SET_GESTURE_TAP_COORDS_MODE, CLEAR_TAP_COORDINATES, DELETE_SAVED_GESTURES_REQUESTED, DELETE_SAVED_GESTURES_DONE,
          SELECT_HOVERED_CENTROID, UNSELECT_HOVERED_CENTROID, SELECT_CENTROID, UNSELECT_CENTROID,
-         SET_SHOW_CENTROIDS, TOGGLE_SHOW_ATTRIBUTES
+         SET_SHOW_CENTROIDS, TOGGLE_SHOW_ATTRIBUTES, TOGGLE_REFRESHING_STATE, SET_SIRI_COMMAND_VALUE
 } from '../actions/Inspector';
 import { SCREENSHOT_INTERACTION_MODE, INTERACTION_MODE, APP_MODE } from '../components/Inspector/shared';
 
@@ -32,13 +32,16 @@ const INITIAL_STATE = {
   lastActiveMoment: null,
   expandedPaths: ['0'],
   isRecording: false,
+  isRefreshingSource: true,
   showRecord: false,
   showBoilerplate: false,
   recordedActions: [],
   actionFramework: DEFAULT_FRAMEWORK,
   sessionDetails: {},
   isGestureEditorVisible: false,
   isLocatorTestModalVisible: false,
+  isSiriCommandModalVisible: false,
+  siriCommandValue: '',
   showCentroids: false,
   locatorTestStrategy: 'id',
   locatorTestValue: '',
@@ -293,6 +296,24 @@ export default function inspector (state = INITIAL_STATE, action) {
         isLocatorTestModalVisible: false,
       };
 
+    case SHOW_SIRI_COMMAND_MODAL:
+      return {
+        ...state,
+        isSiriCommandModalVisible: true,
+      };
+
+    case HIDE_SIRI_COMMAND_MODAL:
+      return {
+        ...state,
+        isSiriCommandModalVisible: false,
+      };
+
+    case SET_SIRI_COMMAND_VALUE:
+      return {
+        ...state,
+        siriCommandValue: action.siriCommandValue
+      };
+
     case SET_LOCATOR_TEST_STRATEGY:
       return {
         ...state,
@@ -591,6 +612,9 @@ export default function inspector (state = INITIAL_STATE, action) {
 
     case TOGGLE_SHOW_ATTRIBUTES:
       return {...state, showSourceAttrs: !state.showSourceAttrs};
+
+    case TOGGLE_REFRESHING_STATE:
+        return {...state, isRefreshingSource: !state.isRefreshingSource};
 
     default:
       return {...state};