Feature: Persist utilization time across Appium processes (#561)

package.json

@@ -1,6 +1,6 @@
 {
   "name": "appium-device-farm",
-  "version": "3.0.0",
+  "version": "3.1.0",
   "description": "An appium 2.0 plugin that manages and create driver session on available devices",
   "main": "./lib/index.js",
   "scripts": {
@@ -21,6 +21,7 @@
     "prettier": "prettier 'src/**/*.ts' 'web/**/*.ts' 'web/**/*.tsx' --write --single-quote",
     "appium-home": "rm -rf rm -rf /tmp/some-temp-dir && export APPIUM_HOME=/tmp/some-temp-dir",
     "install-plugin": "npm run build && appium plugin install --source=local $(pwd)",
+    "clear-cache": "rm -rf $HOME/.cache/appium-device-farm",
     "reinstall-plugin": "APPIUM_HOME=/tmp/some-temp-dir && npm run appium-home && (appium plugin uninstall device-farm || exit 0) && npm run install-plugin",
     "run-server": "APPIUM_HOME=/tmp/some-temp-dir appium server -ka 800 --config=./server-config.json --use-plugins=device-farm -pa /wd/hub"
   },
@@ -46,6 +47,7 @@
   "dependencies": {
     "@appium/base-plugin": "1.10.5",
     "@appium/types": "0.5.0",
+    "@types/node-persist": "^3.1.3",
     "appium-adb": "^9.5.0",
     "appium-chromedriver": "^5.2.1",
     "appium-ios-device": "^2.4.1",
@@ -59,6 +61,7 @@
     "lodash": "^4.17.21",
     "lokijs": "^1.5.12",
     "node-cache": "^5.1.2",
+    "node-persist": "^3.1.0",
     "node-schedule": "^2.0.0",
     "node-simctl": "^7.0.0",
     "ora": "5.4.1",

src/data-service/device-service.ts

@@ -2,6 +2,7 @@ import { DeviceModel } from './db';
 import { IDevice } from '../interfaces/IDevice';
 import { IDeviceFilterOptions } from '../interfaces/IDeviceFilterOptions';
 import logger from '../logger';
+import { setUtilizationTime } from '../device-utils';
 
 export function saveDevices(devices: Array<IDevice>): any {
   const connectedDeviceIds = new Set(devices.map((device) => device.udid));
@@ -115,6 +116,7 @@ export function unblockDevice(sessionId: string) {
   const currentTime = new Date().getTime();
   const utilization = currentTime - sessionStart;
   const totalUtilization = device.totalUtilizationTimeMilliSec + utilization;
+  setUtilizationTime(device.udid, totalUtilization);
   DeviceModel.chain()
     .find({
       session_id: sessionId,

src/device-managers/AndroidDeviceManager.ts

@@ -8,6 +8,7 @@ import { fs } from '@appium/support';
 import { DeviceFactory } from './factory/DeviceFactory';
 import ChromeDriverManager from './ChromeDriverManager';
 import { Container } from 'typedi';
+import { getUtilizationTime } from '../device-utils';
 
 export default class AndroidDeviceManager implements IDeviceManager {
   private adb: any;
@@ -74,6 +75,7 @@ export default class AndroidDeviceManager implements IDeviceManager {
         } else {
           log.info(`Android Device details for ${device.udid} not available. So querying now.`);
           const systemPort = await getFreePort();
+          const totalUtilizationTimeMilliSec = await getUtilizationTime(device.udid);
           const [sdk, realDevice, name, chromeDriverPath] = await Promise.all([
             this.getDeviceVersion(device.udid),
             this.isRealDevice(device.udid),
@@ -92,7 +94,7 @@ export default class AndroidDeviceManager implements IDeviceManager {
             platform: 'android',
             deviceType: realDevice ? 'real' : 'emulator',
             host: `http://127.0.0.1:${cliArgs.port}`,
-            totalUtilizationTimeMilliSec: 0,
+            totalUtilizationTimeMilliSec: totalUtilizationTimeMilliSec,
             sessionStartTime: 0,
             chromeDriverPath,
           });

src/device-managers/IOSDeviceManager.ts

@@ -10,6 +10,7 @@ import axios from 'axios';
 import { DeviceFactory } from './factory/DeviceFactory';
 import os from 'os';
 import path from 'path';
+import { getUtilizationTime } from '../device-utils';
 
 export default class IOSDeviceManager implements IDeviceManager {
   /**
@@ -97,6 +98,7 @@ export default class IOSDeviceManager implements IDeviceManager {
         log.info(`IOS Device details for ${udid} not available. So querying now.`);
         const wdaLocalPort = await getFreePort();
         const mjpegServerPort = await getFreePort();
+        const totalUtilizationTimeMilliSec = await getUtilizationTime(udid);
         const [sdk, name] = await Promise.all([this.getOSVersion(udid), this.getDeviceName(udid)]);
         deviceState.push(
           Object.assign({
@@ -110,7 +112,7 @@ export default class IOSDeviceManager implements IDeviceManager {
             deviceType: 'real',
             platform: 'ios',
             host: `http://127.0.0.1:${cliArgs.port}`,
-            totalUtilizationTimeMilliSec: 0,
+            totalUtilizationTimeMilliSec: totalUtilizationTimeMilliSec,
             sessionStartTime: 0,
             derivedDataPath: path.join(
               os.homedir(),
@@ -172,6 +174,7 @@ export default class IOSDeviceManager implements IDeviceManager {
     await asyncForEach(buildSimulators, async (device: IDevice) => {
       const wdaLocalPort = await getFreePort();
       const mjpegServerPort = await getFreePort();
+      const totalUtilizationTimeMilliSec = await getUtilizationTime(device.udid);
       simulators.push(
         Object.assign({
           ...device,
@@ -182,7 +185,7 @@ export default class IOSDeviceManager implements IDeviceManager {
           platform: 'ios',
           deviceType: 'simulator',
           host: `http://127.0.0.1:${cliArgs.port}`,
-          totalUtilizationTimeMilliSec: 0,
+          totalUtilizationTimeMilliSec: totalUtilizationTimeMilliSec,
           sessionStartTime: 0,
           derivedDataPath: path.join(
             os.homedir(),

src/device-utils.ts

@@ -20,6 +20,10 @@ import logger from './logger';
 import CapabilityFactory from './device-managers/factory/CapabilityFactory';
 import DevicePlatform from './enums/Platform';
 import _ from 'lodash';
+import os from 'os';
+import fs from 'fs';
+import path from 'path';
+import { LocalStorage } from 'node-persist';
 
 const DEVICE_AVAILABILITY_TIMEOUT = 180000;
 const DEVICE_AVAILABILITY_QUERY_INTERVAL = 10000;
@@ -113,6 +117,47 @@ export async function updateCapabilityForDevice(capability: any, device: IDevice
   }
 }
 
+/**
+ * Sets up node-persist storage in local cache
+ * @returns storage
+ */
+export async function initlializeStorage() {
+  const basePath = path.join(os.homedir(), ".cache", "appium-device-farm", "storage");
+  await fs.promises.mkdir(basePath, { recursive: true });
+  const storage = require('node-persist');
+  const localStorage = storage.create({dir: basePath});
+  await localStorage.init();
+  Container.set('LocalStorage', localStorage);
+}
+
+function getStorage() {
+  return Container.get('LocalStorage') as LocalStorage;
+}
+
+
+/**
+ * Gets utlization time for a device from storage
+ * Returns 0 if the device has not been used an thus utilization time has not been saved
+ * @param udid
+ * @returns number
+ */
+export async function getUtilizationTime(udid: string) {
+  const value = await getStorage().getItem(udid);
+  if (value !== undefined) {
+    return value;
+  }
+  return 0;
+}
+
+/**
+ * Sets utilization time for a device to storage
+ * @param udid 
+ * @param utilizationTime 
+ */
+export async function setUtilizationTime(udid: string, utilizationTime: number) {
+  await getStorage().setItem(udid, utilizationTime);
+}
+
 /**
  * Method to get the device filters from the custom session capability
  * This filter will be used as in the query to find the free device from the databse

src/plugin.ts

@@ -15,6 +15,7 @@ import {
   refreshDeviceList,
   cronReleaseBlockedDevices,
   allocateDeviceForSession,
+  initlializeStorage,
 } from './device-utils';
 import { DeviceFarmManager } from './device-managers';
 import { Container } from 'typedi';
@@ -29,6 +30,8 @@ import ora from 'ora';
 import { hubUrl } from './helpers';
 import Cloud from './enums/Cloud';
 import ChromeDriverManager from './device-managers/ChromeDriverManager';
+import { LocalStorage } from 'node-persist';
+
 const commandsQueueGuard = new AsyncLock();
 const DEVICE_MANAGER_LOCK_NAME = 'DeviceManager';
 
@@ -75,6 +78,7 @@ class DevicePlugin extends BasePlugin {
     });
     Container.set(DeviceFarmManager, deviceManager);
     if (chromeDriverManager) Container.set(ChromeDriverManager, chromeDriverManager);
+    await initlializeStorage();
     logger.info(
       `📣📣📣 Device Farm Plugin will be served at 🔗 http://localhost:${cliArgs.port}/device-farm`
     );

test/integration/androidDevices.spec.js

@@ -2,7 +2,7 @@ import { expect } from 'chai';
 import { DeviceFarmManager } from '../../src/device-managers';
 import { Container } from 'typedi';
 import { DeviceModel } from '../../src/data-service/db';
-import { updateDeviceList, allocateDeviceForSession } from '../../src/device-utils';
+import { updateDeviceList, allocateDeviceForSession, initlializeStorage } from '../../src/device-utils';
 
 const cliArgs = {
   platform: 'android',
@@ -14,6 +14,7 @@ const cliArgs = {
 };
 describe('Android Test', () => {
   it('Allocate free device and verify the device state is busy in db', async () => {
+    await initlializeStorage();
     const deviceManager = new DeviceFarmManager(cliArgs);
     Container.set(DeviceFarmManager, deviceManager);
     await updateDeviceList();
@@ -32,6 +33,7 @@ describe('Android Test', () => {
   });
 
   it('Allocate second free device and verify both the device state is busy in db', async () => {
+    await initlializeStorage();
     const deviceManager = new DeviceFarmManager(cliArgs);
     Container.set(DeviceFarmManager, deviceManager);
     await updateDeviceList();
@@ -50,6 +52,7 @@ describe('Android Test', () => {
   });
 
   it('Finding a device should throw error when all devices are busy', async () => {
+    await initlializeStorage();
     const deviceManager = new DeviceFarmManager(cliArgs);
     Container.set(DeviceFarmManager, deviceManager);
     await updateDeviceList();

test/integration/iOSDevices.spec.js

@@ -2,10 +2,11 @@ import { expect } from 'chai';
 import { DeviceFarmManager } from '../../src/device-managers';
 import { Container } from 'typedi';
 
-import { updateDeviceList, allocateDeviceForSession } from '../../src/device-utils';
+import { updateDeviceList, allocateDeviceForSession, initlializeStorage } from '../../src/device-utils';
 
 describe('IOS Test', () => {
   it('Throw error when no device is found for given capabilities', async () => {
+    await initlializeStorage();
     const deviceManager = new DeviceFarmManager({
       platform: 'iOS',
       deviceTypes: 'both',

test/integration/iOSSimulator.spec.js

@@ -2,7 +2,7 @@ import { expect } from 'chai';
 import { DeviceFarmManager } from '../../src/device-managers';
 import { Container } from 'typedi';
 
-import { updateDeviceList, allocateDeviceForSession } from '../../src/device-utils';
+import { updateDeviceList, allocateDeviceForSession, initlializeStorage } from '../../src/device-utils';
 import { DeviceModel } from '../../src/data-service/db';
 
 import Simctl from 'node-simctl';
@@ -13,6 +13,7 @@ const name = 'My Device Name';
 
 describe('IOS Simulator Test', () => {
   it('Should find free iPhone simulator when app path has .app extension and set busy status to true', async () => {
+    await initlializeStorage();
     const deviceManager = new DeviceFarmManager({
       platform: 'ios',
       deviceTypes: 'both',
@@ -46,6 +47,7 @@ describe('IOS Simulator Test', () => {
   });
 
   it('Should find free iPad simulator when app path has .app extension and set busy status to true', async () => {
+    await initlializeStorage();
     const deviceManager = new DeviceFarmManager({
       platform: 'ios',
       deviceTypes: 'both',
@@ -81,6 +83,7 @@ describe('Boot simulator test', async () => {
   });
 
   it('Should pick Booted simulator when app path has .app', async () => {
+    await initlializeStorage();
     const deviceManager = new DeviceFarmManager({
       platform: 'ios',
       deviceTypes: 'both',

test/unit/AndroidDeviceManager.spec.js

@@ -2,6 +2,7 @@ import sinon from 'sinon';
 import { expect } from 'chai';
 import AndroidDeviceManager from '../../src/device-managers/AndroidDeviceManager';
 import * as Helper from '../../src/helpers';
+import * as DeviceUtils from '../../src/device-utils';
 var sandbox = sinon.createSandbox();
 
 const cliArgs = {
@@ -32,6 +33,7 @@ describe('Android Device Manager', function () {
     realDevice.onFirstCall().returns(false);
     realDevice.onSecondCall().returns(true);
     sandbox.stub(Helper, 'getFreePort').returns(54321);
+    sandbox.stub(DeviceUtils, 'getUtilizationTime').returns(0);
     const devices = await androidDevices.getDevices('both', [], { port: 4723, plugin: cliArgs });
     expect(devices).to.deep.equal([
       {
@@ -82,6 +84,7 @@ describe('Android Device Manager', function () {
     realDevice.onFirstCall().returns(false);
     realDevice.onSecondCall().returns(true);
     sandbox.stub(Helper, 'getFreePort').returns(54321);
+    sandbox.stub(DeviceUtils, 'getUtilizationTime').returns(0);
     const devices = await androidDevices.getDevices('simulated', [], {
       port: 4723,
       plugin: cliArgs,
@@ -120,6 +123,7 @@ describe('Android Device Manager', function () {
     realDevice.onFirstCall().returns(false);
     realDevice.onSecondCall().returns(true);
     sandbox.stub(Helper, 'getFreePort').returns(54322);
+    sandbox.stub(DeviceUtils, 'getUtilizationTime').returns(0);
     const devices = await androidDevices.getDevices('real', [], { port: 4723, plugin: cliArgs });
     expect(devices).to.deep.equal([
       {

test/unit/IOSDeviceManager.spec.js

@@ -2,6 +2,7 @@ import sinon from 'sinon';
 import { expect } from 'chai';
 import IOSDeviceManager from '../../src/device-managers/IOSDeviceManager';
 import * as Helper from '../../src/helpers';
+import * as DeviceUtils from '../../src/device-utils';
 import os from 'os';
 import path from 'path';
 import { deviceMock } from './fixtures/devices';
@@ -24,6 +25,7 @@ describe('IOS Device Manager', () => {
     sandbox.stub(iosDevices, 'getOSVersion').returns('14.1.1');
     sandbox.stub(Helper, 'isMac').returns(true);
     sandbox.stub(Helper, 'getFreePort').returns(54093);
+    sandbox.stub(DeviceUtils, 'getUtilizationTime').returns(0);
     sandbox.stub(iosDevices, 'getDeviceName').returns('Sai’s iPhone');
     sandbox.stub(iosDevices, 'getSimulators').returns([
       {
@@ -105,6 +107,7 @@ describe('IOS Device Manager', () => {
     sandbox.stub(iosDeviceManager, 'getOSVersion').returns('14.1.1');
     sandbox.stub(iosDeviceManager, 'getDeviceName').returns('Sai’s iPhone');
     sandbox.stub(Helper, 'getFreePort').returns(54093);
+    sandbox.stub(DeviceUtils, 'getUtilizationTime').returns(0);
     sandbox.stub(iosDeviceManager, 'getLocalSims').returns(deviceMock);
     const devices = await iosDeviceManager.getDevices('both', [], { port: 4723, plugin: cliArgs });
     expect(devices.length).to.be.equal(3);
@@ -133,6 +136,7 @@ describe('IOS Device Manager', () => {
     let iosDeviceManager = new IOSDeviceManager();
     sandbox.stub(Helper, 'getFreePort').returns(54093);
     sandbox.stub(iosDeviceManager, 'getLocalSims').returns(deviceMock);
+    sandbox.stub(DeviceUtils, 'getUtilizationTime').returns(0);
     const devices = await iosDeviceManager.getDevices('simulated', [], {
       port: 4723,
       plugin: cliArgs,
@@ -148,6 +152,7 @@ describe('IOS Device Manager', () => {
     sandbox.stub(iosDevices, 'getOSVersion').returns('14.1.1');
     sandbox.stub(iosDevices, 'getDeviceName').returns('Sai’s iPhone');
     sandbox.stub(Helper, 'getFreePort').returns(54093);
+    sandbox.stub(DeviceUtils, 'getUtilizationTime').returns(0);
     sandbox.stub(iosDevices, 'getSimulators').returns([
       {
         name: 'iPad Air (3rd generation)',
@@ -195,6 +200,7 @@ describe('IOS Device Manager', () => {
     sandbox.stub(iosDevices, 'getOSVersion').returns('14.1.1');
     sandbox.stub(iosDevices, 'getDeviceName').returns('Sai’s iPhone');
     sandbox.stub(Helper, 'getFreePort').returns(54093);
+    sandbox.stub(DeviceUtils, 'getUtilizationTime').returns(0);
     sandbox.stub(iosDevices, 'getSimulators').returns([
       {
         name: 'iPad Air (3rd generation)',
@@ -224,6 +230,7 @@ describe('IOS Device Manager', () => {
     sandbox.stub(iosDevices, 'getOSVersion').returns('14.1.1');
     sandbox.stub(iosDevices, 'getDeviceName').returns('Sai’s iPhone');
     sandbox.stub(Helper, 'getFreePort').returns(54093);
+    sandbox.stub(DeviceUtils, 'getUtilizationTime').returns(0);
     sandbox.stub(iosDevices, 'getSimulators').returns([
       {
         name: 'iPad Air (3rd generation)',

test/unit/RemoteAndroid.spec.js

@@ -1,6 +1,7 @@
 import Sinon from 'sinon';
 import AndroidDeviceManager from '../../src/device-managers/AndroidDeviceManager';
 import * as Helper from '../../src/helpers';
+import * as DeviceUtils from '../../src/device-utils';
 import { expect } from 'chai';
 import axios from 'axios';
 const firstNode = 'http://192.168.0.103';
@@ -55,6 +56,7 @@ describe('Remote Android', () => {
     sandbox.stub(androidDevices, 'getChromeVersion').returns('/var/path/chromedriver');
     sandbox.stub(androidDevices, 'isRealDevice').returns(false);
     sandbox.stub(Helper, 'getFreePort').returns(54321);
+    sandbox.stub(DeviceUtils, 'getUtilizationTime').returns(0);
     const devices = await androidDevices.getDevices('both', [], { port: 4723, plugin: cliArgs });
     const expected = [
       {

test/unit/RemoteIOs.spec.js

@@ -2,6 +2,7 @@ import ip from 'ip';
 import Sinon from 'sinon';
 import IOSDeviceManager from '../../src/device-managers/IOSDeviceManager';
 import * as Helper from '../../src/helpers';
+import * as DeviceUtils from '../../src/device-utils';
 import { expect } from 'chai';
 import axios from 'axios';
 import os from 'os';
@@ -71,6 +72,7 @@ describe('Remote IOS', () => {
     sandbox.stub(iosDevices, 'getOSVersion').returns('14.1.1');
     sandbox.stub(iosDevices, 'getDeviceName').returns('Sai’s iPhone');
     sandbox.stub(Helper, 'getFreePort').returns(54093);
+    sandbox.stub(DeviceUtils, 'getUtilizationTime').returns(0);
     sandbox
       .stub(iosDevices, 'fetchLocalSimulators')
       .withArgs(simulators, cliArgs)