test: Integrate automated smoke test into release workflow (#153)

aws-observability · May 19, 2022 · f291142 · f291142
1 parent 3dd112f
commit f291142
Show file tree

Hide file tree

Showing 11 changed files with 427 additions and 111 deletions.
diff --git a/.github/scripts/update_smoke_test.sh b/.github/scripts/update_smoke_test.sh
@@ -0,0 +1,11 @@
+MONITOR_ID=$1
+REGION=$2
+GUEST_ARN=$3
+IDENTITY_POOL=$4
+ENDPOINT=$5
+CDN=$6
+VERSION=$(npm pkg get version | sed 's/"//g')/cwr.js
+CDN+=${VERSION}
+awk '{sub(/\$MONITOR_ID/,MONITOR_ID);sub(/\$REGION/,REGION);sub(/\$CDN/,CDN);sub(/\$GUEST_ARN/,GUEST_ARN);sub(/\$IDENTITY_POOL/,IDENTITY_POOL);sub(/\$ENDPOINT/,ENDPOINT);}1' \
+ MONITOR_ID="'$MONITOR_ID'" REGION="'$REGION'" CDN="'$CDN'" GUEST_ARN="'$GUEST_ARN'" IDENTITY_POOL="'$IDENTITY_POOL'" ENDPOINT="'$ENDPOINT'" app/smoke.html
+
diff --git a/.github/scripts/upload_smoke_test.sh b/.github/scripts/upload_smoke_test.sh
@@ -0,0 +1,3 @@
+bucket=$1
+key=smoke-$(npm pkg get version | sed 's/"//g').html
+aws s3api put-object --bucket $bucket --key "$key" --body processed_smoke.html --content-type "text/html"
diff --git a/.github/workflows/cd.yaml b/.github/workflows/cd.yaml
@@ -19,7 +19,7 @@ jobs:
                   node-version: '16.x'
                   registry-url: 'https://registry.npmjs.org'
 
-            - name: Fetch AWS Credentials
+            - name: Fetch AWS Credentials for Deployment
               run: |
                   export AWS_ROLE_ARN=${{ secrets.ROLE }}
                   export AWS_WEB_IDENTITY_TOKEN_FILE=/tmp/awscreds
@@ -50,6 +50,41 @@ jobs:
                   chmod u+x .github/scripts/deploy.sh
                   .github/scripts/deploy.sh ${{ secrets.BUCKET }}
 
+            - name: Fetch AWS Credentials for Smoke Test
+              run: |
+                  export AWS_ROLE_ARN=${{ secrets.SMOKE_TEST_ROLE }}
+                  export AWS_WEB_IDENTITY_TOKEN_FILE=/tmp/awscreds
+                  export AWS_DEFAULT_REGION=us-east-1
+
+                  echo AWS_WEB_IDENTITY_TOKEN_FILE=$AWS_WEB_IDENTITY_TOKEN_FILE >> $GITHUB_ENV
+                  echo AWS_ROLE_ARN=$AWS_ROLE_ARN >> $GITHUB_ENV
+                  echo AWS_DEFAULT_REGION=$AWS_DEFAULT_REGION >> $GITHUB_ENV
+
+                  curl -H "Authorization: bearer $ACTIONS_ID_TOKEN_REQUEST_TOKEN" "$ACTIONS_ID_TOKEN_REQUEST_URL" | jq -r '.value' > $AWS_WEB_IDENTITY_TOKEN_FILE
+
+            - name: Update Smoke Test Application
+              id: update-smoke-test
+              run: |
+                  chmod u+x .github/scripts/update_smoke_test.sh
+                  .github/scripts/update_smoke_test.sh ${{ secrets.SMOKE_MONITOR }} ${{ secrets.SMOKE_REGION }} ${{ secrets.SMOKE_ARN }} ${{ secrets.SMOKE_IDENTITY }} ${{ secrets.CONFIG_ENDPOINT }} ${{ secrets.CDN }} >> processed_smoke.html
+
+            - name: Upload Smoke Test to CloudFront
+              id: upload-smoke-test
+              run: |
+                  chmod u+x .github/scripts/upload_smoke_test.sh
+                  .github/scripts/upload_smoke_test.sh ${{ secrets.SMOKE_BUCKET }}
+
+            - name: Install PlayWright
+              run: npx playwright install --with-deps chromium
+
+            - name: Run Smoke Test
+              env:
+                  URL: ${{ secrets.SMOKE_URL }}
+                  MONITOR: ${{ secrets.SMOKE_MONITOR }}
+                  ENDPOINT: ${{ secrets.SMOKE_ENDPOINT }}
+                  NAME: ${{ secrets.SMOKE_MONITOR_NAME }}
+              run: npm run smoke:headless
+
             - name: Publish to NPM
               run: npm publish
               env:

diff --git a/.gitignore b/.gitignore
@@ -15,6 +15,6 @@ chromedriver.log
 geckodriver.log
 safaridriver.log
 tests_output
-app/smoke.html
 logs
 .idea
+app/smoke_local.html
diff --git a/app/smoke.html b/app/smoke.html
@@ -0,0 +1,221 @@
+<!DOCTYPE html>
+<html>
+    <head>
+        <!-- <meta
+            http-equiv="Content-Security-Policy"
+            content="default-src *; script-src client.rum.us-east-1.amazonaws.com 'nonce-smokeTest'; style-src 'nonce-smokeTest';"
+        /> -->
+        <title>RUM Smoke Test</title>
+        <script>
+            (function (n, i, v, r, s, c, x, z) {
+                x = window.AwsRumClient = {
+                    q: [],
+                    n: n,
+                    i: i,
+                    v: v,
+                    r: r,
+                    c: c
+                };
+                window[n] = function (c, p) {
+                    x.q.push({ c: c, p: p });
+                };
+                z = document.createElement('script');
+                z.async = true;
+                z.src = s;
+                document.head.insertBefore(
+                    z,
+                    document.getElementsByTagName('script')[0]
+                );
+            })('cwr', $MONITOR_ID, '1.0.0', $REGION, $CDN, {
+                sessionSampleRate: 1,
+                guestRoleArn: $GUEST_ARN,
+                identityPoolId: $IDENTITY_POOL,
+                endpoint: $ENDPOINT,
+                telemetries: ['performance', 'errors', 'http'],
+                allowCookies: true,
+                enableXRay: false
+            });
+        </script>
+
+        <link
+            rel="icon"
+            type="image/png"
+            href="https://awsmedia.s3.amazonaws.com/favicon.ico"
+        />
+
+        <script nonce="smokeTest">
+            function dispatch() {
+                cwr('dispatch');
+            }
+
+            function clearRequestResponse() {
+                document.getElementById('request_url').innerText = '';
+                document.getElementById('request_header').innerText = '';
+                document.getElementById('request_body').innerText = '';
+
+                document.getElementById('response_status').innerText = '';
+                document.getElementById('response_header').innerText = '';
+                document.getElementById('response_body').innerText = '';
+            }
+
+            // Specific to JavaScript error plugin
+            function triggerTypeError() {
+                undefined.foo();
+            }
+
+            function throwErrorString() {
+                throw 'thrown string';
+            }
+
+            function recordCaughtError() {
+                cwr('recordError', new Error('My error message'));
+            }
+
+            function disable() {
+                cwr('disable');
+            }
+
+            function enable() {
+                cwr('enable');
+            }
+
+            // //////////////////////////
+
+            function pushStateOneToHistory() {
+                window.history.pushState(
+                    { state: 'one' },
+                    'Page One',
+                    '/page_view_one?search=foo#hash1'
+                );
+            }
+            function pushStateTwoToHistory() {
+                window.history.pushState(
+                    { state: 'two' },
+                    'Page Two',
+                    '/page_view_two?search=bar#hash2'
+                );
+            }
+
+            function replaceState() {
+                window.history.replaceState(
+                    { state: 'one' },
+                    'Page Ten',
+                    '/page_view_Ten?search=bar#asdf'
+                );
+            }
+
+            function defaultState() {
+                window.history.replaceState(
+                    { state: 'one' },
+                    'Page Ten',
+                    '/page_event.html'
+                );
+            }
+
+            function back() {
+                window.history.back();
+            }
+
+            function forward() {
+                window.history.forward();
+            }
+
+            function go(number) {
+                window.history.go(number);
+            }
+
+            function recordPageView() {
+                cwr('recordPageView', '/page_view_two');
+            }
+
+            function httpStatFetch500() {
+                fetch('https://httpstat.us/500');
+            }
+
+            function httpStatFetch200() {
+                fetch('https://httpstat.us/200');
+            }
+
+            function httpStatXhr500() {
+                let xhr = new XMLHttpRequest();
+                xhr.open('GET', 'https://httpstat.us/500', true);
+                xhr.addEventListener('load', function (e) {
+                    console.log(xhr.response);
+                });
+                xhr.send();
+            }
+
+            function httpStatXhr200() {
+                let xhr = new XMLHttpRequest();
+                xhr.open('GET', 'https://httpstat.us/200', true);
+                xhr.addEventListener('load', function (e) {
+                    console.log(xhr.response);
+                });
+                xhr.send();
+            }
+        </script>
+
+        <style nonce="smokeTest">
+            table {
+                border-collapse: collapse;
+                margin-top: 10px;
+                margin-bottom: 10px;
+            }
+
+            td,
+            th {
+                border: 1px solid black;
+                text-align: left;
+                padding: 8px;
+            }
+        </style>
+    </head>
+    <body>
+        <p id="welcome">This application is used for RUM smoke testing.</p>
+        <hr />
+        <button id="pushStateOneToHistory" onclick="pushStateOneToHistory()">
+            Push State One to History
+        </button>
+        <button id="pushStateTwoToHistory" onclick="pushStateTwoToHistory()">
+            Push State Two to History
+        </button>
+        <button id="replaceState" onclick="replaceState()">
+            Replace current state in History
+        </button>
+        <button id="replaceDefault" onclick="defaultState()">
+            Return to default
+        </button>
+        <button id="back" onclick="back()">Back</button>
+        <button id="forward" onclick="forward()">Forward</button>
+        <button id="go-back" onclick="go(-2)">Go (back two pages)</button>
+        <button id="go-forward" onclick="go(2)">Go (forward two pages)</button>
+        <hr />
+        <button id="triggerTypeError" onclick="triggerTypeError()">
+            Trigger TypeError
+        </button>
+        <button id="throwErrorString" onclick="throwErrorString()">
+            Throw error string
+        </button>
+        <button id="recordCaughtError" onclick="recordCaughtError()">
+            Record caught error
+        </button>
+        <hr />
+        <button id="httpStatFetch200" onclick="httpStatFetch200()">
+            httpstat fetch 200
+        </button>
+        <button id="httpStatFetch500" onclick="httpStatFetch500()">
+            httpstat fetch 500
+        </button>
+        <button id="httpStatXhr200" onclick="httpStatXhr200()">
+            httpstat xhr 200
+        </button>
+        <button id="httpStatXhr500" onclick="httpStatXhr500()">
+            httpstat xhr 500
+        </button>
+        <hr />
+        <button id="disable" onclick="disable()">Disable</button>
+        <button id="enable" onclick="enable()">Enable</button>
+        <button id="dummyButton">Dummy Button</button>
+        <hr />
+    </body>
+</html>
diff --git a/package.json b/package.json
@@ -51,10 +51,10 @@
         "preinteg:local:nightwatch:firefox": "http-server ./build/dev -s &",
         "integ:local:nightwatch:firefox": "nightwatch -e firefox",
         "postinteg:local:nightwatch:firefox": "kill $(lsof -t -i:8080)",
-        "smoke:local:headless": "cross-env URL=$URL MONITOR_ID=$MONITOR ENDPOINT=$ENDPOINT NAME=$NAME npx playwright test --config=playwright.local.config.ts",
-        "smoke:local": "cross-env URL=$URL MONITOR_ID=$MONITOR ENDPOINT=$ENDPOINT NAME=$NAME npx playwright test --config=playwright.local.config.ts --headed",
-        "smoke": "cross-env URL=$URL MONITOR_ID=$MONITOR ENDPOINT=$ENDPOINT NAME=$NAME npx playwright test --headed",
-        "smoke:headless": "cross-env URL=$URL MONITOR_ID=$MONITOR ENDPOINT=$ENDPOINT NAME=$NAME npx playwright test",
+        "smoke:local:headless": "cross-env URL=$URL MONITOR_ID=$MONITOR ENDPOINT=$ENDPOINT NAME=$NAME VERSION=$npm_package_version npx playwright test --config=playwright.local.config.ts",
+        "smoke:local": "cross-env URL=$URL MONITOR_ID=$MONITOR ENDPOINT=$ENDPOINT NAME=$NAME VERSION=$npm_package_version npx playwright test --config=playwright.local.config.ts --headed",
+        "smoke": "cross-env URL=$URL MONITOR_ID=$MONITOR ENDPOINT=$ENDPOINT NAME=$NAME VERSION=$npm_package_version npx playwright test --config=playwright.config.ts --headed",
+        "smoke:headless": "cross-env URL=$URL MONITOR_ID=$MONITOR ENDPOINT=$ENDPOINT NAME=$NAME VERSION=$npm_package_version npx playwright test --config=playwright.config.ts",
         "prepare": "husky install"
     },
     "devDependencies": {

diff --git a/playwright.config.ts b/playwright.config.ts
@@ -4,6 +4,8 @@ const { devices } = require('@playwright/test');
 
 const config = {
     forbidOnly: !!process.env.CI,
+    reporter: 'list',
+    workers: process.env.CI ? 4 : undefined,
     testDir: 'src/__smoke-test__',
     retries: process.env.CI ? 2 : 2,
     timeout: 300000,

diff --git a/playwright.local.config.ts b/playwright.local.config.ts
@@ -5,6 +5,7 @@ const { devices } = require('@playwright/test');
 const config = {
     forbidOnly: !!process.env.CI,
     testDir: 'src/__smoke-test__',
+    reporter: 'list',
     retries: process.env.CI ? 2 : 2,
     timeout: 300000,
     webServer: {

diff --git a/src/__smoke-test__/dataplane-integ.spec.ts b/src/__smoke-test__/dataplane-integ.spec.ts
@@ -1,4 +1,9 @@
 import { test, expect } from '@playwright/test';
+import {
+    getEventsByType,
+    getUrl,
+    isDataPlaneRequest
+} from 'test-utils/smoke-test-utils';
 import {
     PERFORMANCE_NAVIGATION_EVENT_TYPE,
     PERFORMANCE_RESOURCE_EVENT_TYPE,
@@ -9,26 +14,9 @@ import {
 // Environment variables set through CLI command
 const ENDPOINT = process.env.ENDPOINT;
 const MONITOR_ID = process.env.MONITOR;
-const TEST_URL = process.env.URL || 'http://localhost:9000/smoke.html';
-
+const TEST_URL = getUrl(process.env.URL, process.env.VERSION);
 const TARGET_URL = ENDPOINT + MONITOR_ID + '/';
 
-function getEventsByType(requestBody, eventType) {
-    return requestBody.RumEvents.filter((e) => e.type === eventType);
-}
-
-/**
- * Returns true if the request is a successful PutRumEvents request
- */
-function isDataPlaneRequest(response): boolean {
-    const request = response.request();
-    return (
-        request.method() === 'POST' &&
-        response.status() === 200 &&
-        response.url() === TARGET_URL
-    );
-}
-
 test('when web client calls PutRumEvents then the response code is 200', async ({
     page
 }) => {
@@ -37,7 +25,7 @@ test('when web client calls PutRumEvents then the response code is 200', async (
 
     // Test will timeout if no successful dataplane request is found
     await page.waitForResponse(async (response) =>
-        isDataPlaneRequest(response)
+        isDataPlaneRequest(response, TARGET_URL)
     );
 });
 
@@ -55,7 +43,7 @@ test('when web client calls PutRumEvents then the payload contains all events',
 
     // Test will timeout if no successful dataplane request is found
     const response = await page.waitForResponse(async (response) =>
-        isDataPlaneRequest(response)
+        isDataPlaneRequest(response, TARGET_URL)
     );
 
     // Parse payload to verify event count