Performance Tests for Login and Home Pages by using Playwright

.buildkite/pipelines/performance/nightly.yml

@@ -6,7 +6,7 @@ steps:
         key: "performance-test-iteration-count"
         hint: "How many times you want to run tests? "
         required: true
-    if: build.env('ITERATION_COUNT_ENV') == null
+    if: build.env('PERF_TEST_COUNT') == null
 
   - label: ":male-mechanic::skin-tone-2: Pre-Build"
     command: .buildkite/scripts/lifecycle/pre_build.sh
@@ -19,10 +19,10 @@ steps:
       queue: c2-16
     key: build
 
-  - label: ":muscle: Performance Tests"
-    command: .buildkite/scripts/steps/functional/performance.sh
+  - label: ":muscle: Performance Tests with Playwright config"
+    command: .buildkite/scripts/steps/functional/performance_playwright.sh
     agents:
-      queue: ci-group-6
+      queue: c2-16
     depends_on: build
 
   - wait: ~

.buildkite/scripts/steps/functional/performance.sh → .buildkite/scripts/steps/functional/performance_playwright.sh

@@ -2,20 +2,22 @@
 
 set -uo pipefail
 
-if [ -z "${ITERATION_COUNT_ENV+x}" ]; then
-  ITERATION_COUNT="$(buildkite-agent meta-data get performance-test-iteration-count)"
+if [ -z "${PERF_TEST_COUNT+x}" ]; then
+  TEST_COUNT="$(buildkite-agent meta-data get performance-test-iteration-count)"
 else
-  ITERATION_COUNT=$ITERATION_COUNT_ENV
+  TEST_COUNT=$PERF_TEST_COUNT
 fi
 
-tput setab 2; tput setaf 0; echo "Performance test will be run at ${BUILDKITE_BRANCH} ${ITERATION_COUNT} times"
+tput setab 2; tput setaf 0; echo "Performance test will be run at ${BUILDKITE_BRANCH} ${TEST_COUNT} times"
 
 cat << EOF | buildkite-agent pipeline upload
 steps:
-  - command: .buildkite/scripts/steps/functional/performance_sub.sh
-    parallelism: "$ITERATION_COUNT"
+  - command: .buildkite/scripts/steps/functional/performance_sub_playwright.sh
+    parallelism: "$TEST_COUNT"
     concurrency: 20
     concurrency_group: 'performance-test-group'
+    agents:
+      queue: c2-16
 EOF
 
 

.buildkite/scripts/steps/functional/performance_sub_playwright.sh

@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/scripts/common/util.sh
+
+.buildkite/scripts/bootstrap.sh
+.buildkite/scripts/download_build_artifacts.sh
+
+echo --- Run Performance Tests with Playwright config
+
+node scripts/es snapshot&
+
+esPid=$!
+
+export PERF_TEST_PHASE=WARMUP
+export TEST_ES_URL=http://elastic:changeme@localhost:9200
+export DONT_START_ES=true
+export DISABLE_APM=true
+
+sleep 120
+
+cd "$XPACK_DIR"
+
+# warmup round 1
+checks-reporter-with-killswitch "Run Performance Tests with Playwright Config (Phase: WARMUP)" \
+  node scripts/functional_tests \
+    --debug --bail \
+    --kibana-install-dir "$KIBANA_BUILD_LOCATION" \
+    --config "test/performance/config.playwright.ts";
+
+export DISABLE_APM=false
+export PERF_TEST_PHASE=TEST
+
+checks-reporter-with-killswitch "Run Performance Tests with Playwright Config (Phase: TEST)" \
+  node scripts/functional_tests \
+    --debug --bail \
+    --kibana-install-dir "$KIBANA_BUILD_LOCATION" \
+    --config "test/performance/config.playwright.ts";
+
+kill "$esPid"

package.json

@@ -830,6 +830,7 @@
     "pbf": "3.2.1",
     "pirates": "^4.0.1",
     "pixelmatch": "^5.1.0",
+    "playwright": "^1.17.1",
     "postcss": "^7.0.32",
     "postcss-loader": "^3.0.0",
     "postcss-prefix-selector": "^1.7.2",

packages/kbn-test/src/functional_tests/tasks.ts

@@ -109,7 +109,9 @@ export async function runTests(options: RunTestsParams) {
 
         let es;
         try {
-          es = await runElasticsearch({ config, options: { ...options, log } });
+          if (process.env.DONT_START_ES !== 'true') {
+            es = await runElasticsearch({ config, options: { ...options, log } });
+          }
           await runKibanaServer({ procs, config, options });
           await runFtr({ configPath, options: { ...options, log } });
         } finally {

test/functional/services/remote/network_profiles.ts

@@ -12,23 +12,11 @@ interface NetworkOptions {
   LATENCY: number;
 }
 
-const sec = 1_000;
-const kB = 1024;
+const sec = 10 ** 3;
+const MBps = 10 ** 6 / 8; // megabyte per second (MB/s) (can be abbreviated as MBps)
 
-// Download (kb/s)	Upload (kb/s)	Latency (ms)
-// https://gist.github.com/theodorosploumis/fd4086ee58369b68aea6b0782dc96a2e
+// Selenium uses B/s (bytes) for network throttling
+// Download (B/s)	Upload (B/s) Latency (ms)
 export const NETWORK_PROFILES: { [key: string]: NetworkOptions } = {
-  DEFAULT: { DOWNLOAD: 5 * kB * sec, UPLOAD: 1 * kB * sec, LATENCY: 0.1 * sec },
-  GPRS: { DOWNLOAD: 0.05 * kB * sec, UPLOAD: 0.02 * kB * sec, LATENCY: 0.5 * sec },
-  MOBILE_EDGE: { DOWNLOAD: 0.24 * kB * sec, UPLOAD: 0.2 * kB * sec, LATENCY: 0.84 * sec },
-  '2G_REGULAR': { DOWNLOAD: 0.25 * kB * sec, UPLOAD: 0.05 * kB * sec, LATENCY: 0.3 * sec },
-  '2G_GOOD': { DOWNLOAD: 0.45 * kB * sec, UPLOAD: 0.15 * kB * sec, LATENCY: 0.15 * sec },
-  '3G_SLOW': { DOWNLOAD: 0.78 * kB * sec, UPLOAD: 0.33 * kB * sec, LATENCY: 0.2 * sec },
-  '3G_REGULAR': { DOWNLOAD: 0.75 * kB * sec, UPLOAD: 0.25 * kB * sec, LATENCY: 0.1 * sec },
-  '3G_GOOD': { DOWNLOAD: 1.5 * kB * sec, UPLOAD: 0.75 * kB * sec, LATENCY: 0.04 * sec },
-  '4G_REGULAR': { DOWNLOAD: 4 * kB * sec, UPLOAD: 3 * kB * sec, LATENCY: 0.02 * sec },
-  DSL: { DOWNLOAD: 2 * kB * sec, UPLOAD: 1 * kB * sec, LATENCY: 0.005 * sec },
-  CABLE_5MBPS: { DOWNLOAD: 5 * kB * sec, UPLOAD: 1 * kB * sec, LATENCY: 0.28 * sec },
-  CABLE_8MBPS: { DOWNLOAD: 8 * kB * sec, UPLOAD: 2 * kB * sec, LATENCY: 0.1 * sec },
-  WIFI: { DOWNLOAD: 30 * kB * sec, UPLOAD: 15 * kB * sec, LATENCY: 0.002 * sec },
+  CLOUD_USER: { DOWNLOAD: 6 * MBps, UPLOAD: 6 * MBps, LATENCY: 0.1 * sec },
 };

test/functional/services/remote/webdriver.ts

@@ -39,6 +39,7 @@ const headlessBrowser: string = process.env.TEST_BROWSER_HEADLESS as string;
 const browserBinaryPath: string = process.env.TEST_BROWSER_BINARY_PATH as string;
 const remoteDebug: string = process.env.TEST_REMOTE_DEBUG as string;
 const certValidation: string = process.env.NODE_TLS_REJECT_UNAUTHORIZED as string;
+const noCache: string = process.env.DISABLE_CACHE as string;
 const SECOND = 1000;
 const MINUTE = 60 * SECOND;
 const NO_QUEUE_COMMANDS = ['getLog', 'getStatus', 'newSession', 'quit'];
@@ -118,6 +119,11 @@ function initChromiumOptions(browserType: Browsers, acceptInsecureCerts: boolean
     options.setChromeBinaryPath(browserBinaryPath);
   }
 
+  if (noCache === '1') {
+    options.addArguments('disk-cache-size', '0');
+    options.addArguments('disk-cache-dir', '/dev/null');
+  }
+
   const prefs = new logging.Preferences();
   prefs.setLevel(logging.Type.BROWSER, logging.Level.ALL);
   options.setUserPreferences(chromiumUserPrefs);
@@ -291,12 +297,12 @@ async function attemptToCreateCommand(
   const { session, consoleLog$ } = await buildDriverInstance();
 
   if (throttleOption === '1' && browserType === 'chrome') {
-    const { KBN_NETWORK_TEST_PROFILE = 'DEFAULT' } = process.env;
+    const { KBN_NETWORK_TEST_PROFILE = 'CLOUD_USER' } = process.env;
 
     const profile =
       KBN_NETWORK_TEST_PROFILE in Object.keys(NETWORK_PROFILES)
         ? KBN_NETWORK_TEST_PROFILE
-        : 'DEFAULT';
+        : 'CLOUD_USER';
 
     const {
       DOWNLOAD: downloadThroughput,
@@ -306,9 +312,16 @@ async function attemptToCreateCommand(
 
     // Only chrome supports this option.
     log.debug(
-      `NETWORK THROTTLED with profile ${profile}: ${downloadThroughput}kbps down, ${uploadThroughput}kbps up, ${latency} ms latency.`
+      `NETWORK THROTTLED with profile ${profile}: ${downloadThroughput} B/s down, ${uploadThroughput} B/s up, ${latency} ms latency.`
     );
 
+    if (noCache) {
+      // @ts-expect-error
+      await session.sendDevToolsCommand('Network.setCacheDisabled', {
+        cacheDisabled: true,
+      });
+    }
+
     // @ts-expect-error
     session.setNetworkConditions({
       offline: false,

x-pack/scripts/functional_tests.js

@@ -91,7 +91,6 @@ const onlyNotInCoverageTests = [
   require.resolve('../test/saved_object_tagging/api_integration/security_and_spaces/config.ts'),
   require.resolve('../test/saved_object_tagging/api_integration/tagging_api/config.ts'),
   require.resolve('../test/examples/config.ts'),
-  require.resolve('../test/performance/config.ts'),
   require.resolve('../test/functional_execution_context/config.ts'),
 ];
 

x-pack/test/performance/config.ts → x-pack/test/performance/config.playwright.ts

@@ -10,15 +10,17 @@ import { FtrConfigProviderContext } from '@kbn/test';
 import { services } from './services';
 import { pageObjects } from './page_objects';
 
-// These "secret" values are intentionally written in the source. We would make the APM server accept annonymous traffic if we could
+// These "secret" values are intentionally written in the source. We would make the APM server accept anonymous traffic if we could
 const APM_SERVER_URL = 'https://2fad4006bf784bb8a54e52f4a5862609.apm.us-west1.gcp.cloud.es.io:443';
 const APM_PUBLIC_TOKEN = 'Q5q5rWQEw6tKeirBpw';
 
 export default async function ({ readConfigFile }: FtrConfigProviderContext) {
   const functionalConfig = await readConfigFile(require.resolve('../functional/config'));
 
+  const testFiles = [require.resolve('./tests/playwright/home.ts')];
+
   return {
-    testFiles: [require.resolve('./tests/index.ts')],
+    testFiles,
     services,
     pageObjects,
     servers: functionalConfig.get('servers'),
@@ -31,21 +33,15 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
     kbnTestServer: {
       ...functionalConfig.get('kbnTestServer'),
       env: {
-        ELASTIC_APM_ACTIVE: 'true',
+        ELASTIC_APM_ACTIVE: process.env.DISABLE_APM === 'true' ? 'false' : 'true',
         ELASTIC_APM_CONTEXT_PROPAGATION_ONLY: 'false',
         ELASTIC_APM_ENVIRONMENT: process.env.CI ? 'ci' : 'development',
         ELASTIC_APM_TRANSACTION_SAMPLE_RATE: '1.0',
         ELASTIC_APM_SERVER_URL: APM_SERVER_URL,
         ELASTIC_APM_SECRET_TOKEN: APM_PUBLIC_TOKEN,
         ELASTIC_APM_GLOBAL_LABELS: Object.entries({
-          ftrConfig: `x-pack/test/performance`,
-          jenkinsJobName: process.env.JOB_NAME,
-          jenkinsBuildNumber: process.env.BUILD_NUMBER,
-          prId: process.env.PR_NUMBER,
-          branch: process.env.GIT_BRANCH,
-          commit: process.env.GIT_COMMIT,
-          mergeBase: process.env.PR_MERGE_BASE,
-          targetBranch: process.env.PR_TARGET_BRANCH,
+          ftrConfig: `x-pack/test/performance/tests/config.playwright`,
+          performancePhase: process.env.PERF_TEST_PHASE,
         })
           .filter(([, v]) => !!v)
           .reduce((acc, [k, v]) => (acc ? `${acc},${k}=${v}` : `${k}=${v}`), ''),

x-pack/test/performance/tests/reporting_dashboard.ts → x-pack/test/performance/tests/ftr/reporting_dashboard.ts

@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import { FtrProviderContext } from '../ftr_provider_context';
+import { FtrProviderContext } from '../../ftr_provider_context';
 
 export default function ({ getService, getPageObject }: FtrProviderContext) {
   const retry = getService('retry');

x-pack/test/performance/tests/playwright/home.ts

@@ -0,0 +1,55 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import Url from 'url';
+import { ChromiumBrowser, Page } from 'playwright';
+import testSetup from './setup';
+import { FtrProviderContext } from '../../ftr_provider_context';
+
+export default function ({ getService }: FtrProviderContext) {
+  describe('perf_login_and_home', () => {
+    const config = getService('config');
+    const kibanaUrl = Url.format({
+      protocol: config.get('servers.kibana.protocol'),
+      hostname: config.get('servers.kibana.hostname'),
+      port: config.get('servers.kibana.port'),
+    });
+
+    let page: Page | null = null;
+    let browser: ChromiumBrowser | null = null;
+
+    before(async () => {
+      const context = await testSetup();
+      page = context.page;
+      browser = context.browser;
+    });
+
+    after(async () => {
+      await browser?.close();
+    });
+
+    it('Go to Kibana login page', async () => {
+      await page?.goto(`${kibanaUrl}`);
+    });
+
+    it('Login to Kibana', async () => {
+      const usernameLocator = page?.locator('[data-test-subj=loginUsername]');
+      const passwordLocator = page?.locator('[data-test-subj=loginPassword]');
+      const submitButtonLocator = page?.locator('[data-test-subj=loginSubmit]');
+
+      await usernameLocator?.type('elastic', { delay: 500 });
+      await passwordLocator?.type('changeme', { delay: 500 });
+      await submitButtonLocator?.click({ delay: 1000 });
+    });
+
+    it('Dismiss Welcome Screen', async () => {
+      await page?.waitForLoadState();
+      const skipButtonLocator = page?.locator('[data-test-subj=skipWelcomeScreen]');
+      await skipButtonLocator?.click({ delay: 1000 });
+      await page?.waitForLoadState('networkidle');
+    });
+  });
+}