Merge pull request #12320 from Expensify/andrew-aws-df

Run e2e performance regression tests on merge into `main` via AWS Device farm

.github/actionlint.yaml

@@ -0,0 +1,4 @@
+# See https://github.com/rhysd/actionlint/blob/main/docs/config.md
+self-hosted-runner:
+  labels:
+    - ubuntu-20.04-64core

.github/actions/composite/configureAwsCredentials/action.yml

@@ -8,6 +8,10 @@ inputs:
   AWS_SECRET_ACCESS_KEY:
     description: 'Secret Access Key to AWS'
     required: true
+  AWS_REGION:
+    description: 'Region for AWS'
+    required: true
+    default: 'us-east-1'
 
 runs:
   using: composite
@@ -18,4 +22,4 @@ runs:
       with:
         aws-access-key-id: ${{ inputs.AWS_ACCESS_KEY_ID }}
         aws-secret-access-key: ${{ inputs.AWS_SECRET_ACCESS_KEY }}
-        aws-region: us-east-1
+        aws-region: ${{ inputs.AWS_REGION }}

.github/scripts/validateActionsAndWorkflows.sh

@@ -41,6 +41,12 @@ done
 
 for ((i=0; i < ${#WORKFLOWS[@]}; i++)); do
   WORKFLOW=${WORKFLOWS[$i]}
+
+    # Skip linting e2e workflow due to bug here: https://github.com/SchemaStore/schemastore/issues/2579
+    if [[ "$WORKFLOW" == './workflows/preDeploy.yml' ]]; then
+      continue
+    fi
+
   ajv -s ./tempSchemas/github-workflow.json -d "$WORKFLOW" --strict=false &
   ASYNC_PROCESSES[${#ACTIONS[@]} + $i]=$!
 done

.github/workflows/preDeploy.yml

@@ -221,3 +221,104 @@ jobs:
             3. Once your PR is deployed to _production_, we start a 7-day timer :alarm_clock:. After it has been on production for 7 days without causing any regressions, then we pay out the Upwork job. :moneybag:
 
             So it might take a while before you're paid for your work, but we typically post multiple new jobs every day, so there's plenty of opportunity. I hope you've had a positive experience contributing to this repo! :blush:
+
+  e2e-tests:
+    name: "Run e2e performance regression tests"
+    runs-on: ubuntu-20.04-64core
+    steps:
+      - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8
+        with:
+          fetch-depth: 0
+
+      - uses: Expensify/App/.github/actions/composite/setupNode@main
+
+      - uses: ruby/setup-ruby@eae47962baca661befdfd24e4d6c34ade04858f7
+        with:
+          ruby-version: '2.7'
+          bundler-cache: true
+
+      # Cache gradle to improve Android build time
+      - name: Gradle cache
+        uses: gradle/gradle-build-action@3fbe033aaae657f011f88f29be9e65ed26bd29ef
+
+      - name: Make zip directory for everything to send to AWS Device Farm
+        run: mkdir zip
+
+      - name: Checkout "Compare" commit
+        run: git checkout ${{ github.event.before }}
+
+      - name: Build "Compare" APK
+        run: npm run android-build-e2e
+
+      - name: Copy "Compare" APK
+        run: cp android/app/build/outputs/apk/e2eRelease/app-e2eRelease.apk zip/app-e2eRelease-compare.apk
+
+      - name: Checkout "Baseline" commit (last release)
+        run: git checkout "$(gh release list --limit 1 | awk '{ print $1 }')"
+
+      - name: Build "Baseline" APK
+        run: npm run android-build-e2e
+
+      - name: Copy "Baseline" APK
+        run: cp android/app/build/outputs/apk/e2eRelease/app-e2eRelease.apk zip/app-e2eRelease-baseline.apk
+
+      - name: Checkout previous branch for source code to run on AWS Device farm
+        run: git checkout -
+
+      - name: Copy e2e code into zip folder
+        run: cp -r tests/e2e zip
+
+      - name: Zip everything in the zip directory up
+        run: zip -qr App.zip ./zip
+
+      - name: Configure AWS Credentials
+        uses: Expensify/App/.github/actions/composite/configureAwsCredentials@main
+        with:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          AWS_REGION: us-west-2
+
+      - name: Schedule AWS Device Farm test run
+        uses: realm/aws-devicefarm/test-application@7b9a91236c456c97e28d384c9e476035d5ea686b
+        with:
+          name: App E2E Performance Regression Tests
+          project_arn: ${{ secrets.AWS_PROJECT_ARN }}
+          device_pool_arn: ${{ secrets.AWS_DEVICE_POOL_ARN }}
+          app_file: zip/app-e2eRelease-baseline.apk
+          app_type: ANDROID_APP
+          test_type: APPIUM_NODE
+          test_package_file: App.zip
+          test_package_type: APPIUM_NODE_TEST_PACKAGE
+          test_spec_file: tests/e2e/TestSpec.yml
+          test_spec_type: APPIUM_NODE_TEST_SPEC
+          remote_src: false
+          file_artifacts: Customer Artifacts.zip
+          cleanup: true
+
+      - name: Unzip AWS Device Farm results
+        run: unzip Customer\ Artifacts.zip
+
+      - name: Set output of AWS Device Farm into GitHub ENV
+        run: |
+          { echo 'OUTPUT<<EOF'; cat ./Host_Machine_Files/\$WORKING_DIRECTORY/output.md; echo 'EOF'; } >> "$GITHUB_ENV"
+
+      - name: Get merged pull request
+        id: getMergedPullRequest
+        # TODO: Point back action actions-ecosystem after https://github.com/actions-ecosystem/action-get-merged-pull-request/pull/223 is merged
+        uses: roryabraham/action-get-merged-pull-request@7a7a194f6ff8f3eef58c822083695a97314ebec1
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Leave output of AWS Device Farm as a PR comment
+        run: |
+          gh pr comment ${{ steps.getMergedPullRequest.outputs.number }} -F ./Host_Machine_Files/\$WORKING_DIRECTORY/output.md
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Check if test failed, if so leave a deploy blocker label
+        if: ${{ !contains(env.OUTPUT, 'There are no entries') }}
+        run: |
+          gh pr edit ${{ steps.getMergedPullRequest.outputs.number }} --add-label 'DeployBlockerCash'
+          gh pr comment ${{ steps.getMergedPullRequest.outputs.number }} -b "@Expensify/mobile-deployers 📣 Please look into this performance regression as it's a deploy blocker."
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -93,4 +93,4 @@ storybook-static
 .jest-cache
 
 # E2E test reports
-e2e/.results/
+tests/e2e/.results/

README.md

@@ -95,7 +95,7 @@ variables referenced here get updated since your local `.env` file is ignored.
 - `ONYX_METRICS` (optional) - Set this to `true` to capture even more performance metrics and see them in Flipper
    see [React-Native-Onyx#benchmarks](https://github.com/Expensify/react-native-onyx#benchmarks) for more information
 - `E2E_TESTING` (optional) - This needs to be set to `true` when running the e2e tests for performance regression testing. 
-   This happens usually automatically, read [this](e2e/README.md) for more information
+   This happens usually automatically, read [this](tests/e2e/README.md) for more information
 
 ----
 

fastlane/Fastfile

@@ -16,7 +16,7 @@ opt_out_usage
 platform :android do
   desc "Generate a new local APK for e2e testing"
   lane :build_e2e do
-    ENV["ENVFILE"]="e2e/.env.e2e"
+    ENV["ENVFILE"]="tests/e2e/.env.e2e"
     ENV["ENTRY_FILE"]="src/libs/E2E/reactNativeLaunchingTest.js"
 
     gradle(

package.json

@@ -37,7 +37,7 @@
     "analyze-packages": "ANALYZE_BUNDLE=true webpack --config config/webpack/webpack.common.js --env envFile=.env.production",
     "symbolicate:android": "npx metro-symbolicate android/app/build/generated/sourcemaps/react/release/index.android.bundle.map",
     "symbolicate:ios": "npx metro-symbolicate main.jsbundle.map",
-    "test:e2e": "node ./e2e/testRunner.js"
+    "test:e2e": "node tests/e2e/testRunner.js"
   },
   "dependencies": {
     "@expensify/react-native-web": "0.18.9",

src/libs/E2E/client.js

@@ -1,5 +1,5 @@
-import Routes from '../../../e2e/server/routes';
-import Config from '../../../e2e/config';
+import Routes from '../../../tests/e2e/server/routes';
+import Config from '../../../tests/e2e/config';
 
 const SERVER_ADDRESS = `http://localhost:${Config.SERVER_PORT}`;
 

src/libs/E2E/reactNativeLaunchingTest.js

@@ -12,7 +12,7 @@ Performance.markStart('regularAppStart');
 import '../../../index';
 Performance.markEnd('regularAppStart');
 
-import E2EConfig from '../../../e2e/config';
+import E2EConfig from '../../../tests/e2e/config';
 import E2EClient from './client';
 
 console.debug('==========================');

e2e/README.md → tests/e2e/README.md

@@ -32,7 +32,7 @@ It will run the tests of a test case multiple time to average out the results.
 
 ## Adding tests
 
-To add a test checkout the [designed guide](./ADDING_TESTS.md).
+To add a test checkout the [designed guide](tests/e2e/ADDING_TESTS.md).
 
 ## Structure
 

tests/e2e/TestSpec.yml

@@ -0,0 +1,26 @@
+version: 0.1
+
+phases:
+  install:
+    commands:
+      # Install correct version of node
+      - export NVM_DIR=$HOME/.nvm
+      - . $NVM_DIR/nvm.sh
+      - nvm install 16.15.1
+      - nvm use 16.15.1
+
+      # Reverse ports using AWS magic
+      - PORT=4723
+      - IP_ADDRESS=$(ip -4 addr show eth0 | grep -Po "(?<=inet\s)\d+(\.\d+){3}")
+      - reverse_values="{\"ip_address\":\"$IP_ADDRESS\",\"local_port\":\"$PORT\",\"remote_port\":\"$PORT\"}"
+      - "curl -H \"Content-Type: application/json\" -X POST -d \"$reverse_values\" http://localhost:31007/reverse_forward_tcp"
+      - adb reverse tcp:$PORT tcp:$PORT
+
+  test:
+    commands:
+      - cd zip
+      - npm install underscore
+      - node e2e/testRunner.js -- --skipInstallDeps --skipBuild
+
+artifacts:
+- $WORKING_DIRECTORY