Add visual regression testing library + tests

.gitignore

@@ -6,4 +6,5 @@ cypress/screenshots/
 **/.DS_Store
 .idea
 
-/.opensearch
+/.opensearch
+**/__diff_output__/

DEVELOPER_GUIDE.md

@@ -27,7 +27,7 @@ You should have a running instance of OpenSearch Dashboards to run these tests a
 
 ### Installation
 
-To install the dependencies run 
+To install the dependencies run
 
 ```
 npm install
@@ -85,7 +85,7 @@ with security:
 $ yarn cypress run-with-security --spec "cypress/integration/core-opensearch-dashboards/opensearch-dashboards/*.js"
 ```
 
-These tests run in headless mode by default. 
+These tests run in headless mode by default.
 
 And you can override certain [cypress config or environment variable](cypress.json) by applying additional cli arguments, for example to override the baseUrl and openSearchUrl to test a remote OpenSearch endpoint:
 
@@ -99,7 +99,6 @@ $ yarn cypress run --spec "cypress/integration/core-opensearch-dashboards/opense
 
 `MANAGED_SERVICE_ENDPOINT`: set to true if tests are running against managed service domains.
 
-
 ## Writing tests
 
 The testing library uses [Cypress](https://www.cypress.io/) as its testing framework and follow its high level folder structure. All tests are written under the `./cypress/integration` folder.
@@ -125,6 +124,19 @@ Tests for plugins that are not a part of the [OpenSearch Dashboards](https://git
         /plugins
             /<YOUR_PLUGIN_NAME>
 ```
+
+### Tests with snapshots
+
+[cypress-image-snapshot](https://github.com/jaredpalmer/cypress-image-snapshot) is a visual regression testing framework that can be used to persist snapshots & compare against them during test runs. This is very useful for checking elements that may not allow for easy validation via HTML elements, such as Vega visualizations which are a single black-box `canvas` element.
+
+Usage:
+
+```
+cy.get('<selector>').matchImageSnapshot('<snapshot-name>')
+```
+
+You use the same `matchImageSnapshot()` command to take new snapshots, or compare against existing ones. You can pass the flag `--env updateSnapshots=true` into the command to create/update them. They will be stored in the `snapshots/` directory. To do comparisons against existing snapshots later on, simply run without the flag. For further details, see the [GitHub repository](https://github.com/jaredpalmer/cypress-image-snapshot).
+
 ### Experimental Features
 
 When writing tests for experimental features, please follow these steps.
@@ -146,16 +158,19 @@ Create a new workflow by referring to [this template](https://github.com/opensea
 To make the build repo enable your experimental feature when spinning up OSD service, make sure that you update [this file](https://github.com/opensearch-project/opensearch-build/blob/main/src/test_workflow/integ_test/service_opensearch_dashboards.py) You could either modify the start command or the yml file. To avoid a potentially long start command, it is preferred to modify the yml file to turn on the feature.
 
 ## General
+
 ### Formatting
 
 `prettier` and `ESLint` is integrated and used to standardize formatting of files, where `prettier` takes care of the code formatting and `ESLint` takes care of the code style. You can check the formatting of all files (new and existing) by running
 
 ```
 $ yarn lint
 ```
+
 and auto fix the formatting of all files (new and existing) by running
+
 ```
 $ yarn lint --fix
 ```
 
-`Husky` precommit hook is used to automatically run `yarn lint`, please fix the files according to lint result before commiting code changes (run `yarn lint --fix` for fixable errors, or manully fix code according to error messages). If you have any doubts on `ESLint` rules, feel free to [open an issue](issues).
+`Husky` precommit hook is used to automatically run `yarn lint`, please fix the files according to lint result before commiting code changes (run `yarn lint --fix` for fixable errors, or manully fix code according to error messages). If you have any doubts on `ESLint` rules, feel free to [open an issue](issues).

cypress/integration/core-opensearch-dashboards/opensearch-dashboards/apps/vis-augmenter/dashboard_spec.js

@@ -11,6 +11,8 @@ import {
 import {
   deleteVisAugmenterData,
   bootstrapDashboard,
+  validateVisSnapshot,
+  setDateRangeTo7Days,
 } from '../../../../../utils/dashboards/vis-augmenter/helpers';
 
 describe('Vis augmenter - existing dashboards work as expected', () => {
@@ -89,6 +91,8 @@ describe('Vis augmenter - existing dashboards work as expected', () => {
         dashboardName,
         visualizationSpecs
       );
+      // Setting viewport so the snapshots in the tests are consistent
+      cy.viewport(1280, 720);
     });
 
     beforeEach(() => {
@@ -105,12 +109,25 @@ describe('Vis augmenter - existing dashboards work as expected', () => {
     });
 
     it('View events option does not exist for any visualization', () => {
+      // Change date range to 7 days so there is less variability in charts
+      // which can cause flakiness (e.g., chart snapshot comparisons).
+      setDateRangeTo7Days();
+
       visualizationNames.forEach((visualizationName) => {
+        // Validating after making each vis full-screen. This is because if there
+        // is a lot of visualizations on the screen, not all may be visible at the
+        // same time on the dashboard - some may require scrolling to be in view.
+        validateVisSnapshot(
+          visualizationName,
+          `${visualizationName}-last-7-days`,
+          true
+        );
         cy.getVisPanelByTitle(visualizationName)
           .openVisContextMenu()
           .getMenuItems()
           .contains('View Events')
           .should('not.exist');
+        cy.getVisPanelByTitle(visualizationName).closeVisContextMenu();
       });
     });
 

cypress/integration/plugins/anomaly-detection-dashboards-plugin/vis_augmenter/associate_detector_spec.js

@@ -14,6 +14,8 @@ import {
   ensureDetectorIsLinked,
   ensureDetectorDetails,
   openDetectorDetailsPageFromFlyout,
+  validateVisSnapshot,
+  setDateRangeTo7Days,
 } from '../../../../utils/helpers';
 import {
   INDEX_PATTERN_FILEPATH_SIMPLE,
@@ -51,6 +53,8 @@ describe('Anomaly detection integration with vis augmenter', () => {
       dashboardName,
       [visualizationSpec]
     );
+    // Setting viewport so the snapshots in the tests are consistent
+    cy.viewport(1280, 720);
   });
 
   after(() => {
@@ -82,6 +86,13 @@ describe('Anomaly detection integration with vis augmenter', () => {
     // the number of features will equal the number of metrics we have specified.
     ensureDetectorDetails(detectorName, visualizationSpec.metrics.length);
 
+    cy.visitDashboard(dashboardName);
+    setDateRangeTo7Days();
+    validateVisSnapshot(
+      visualizationName,
+      `${visualizationName}-with-created-detector`
+    );
+
     unlinkDetectorFromVis(dashboardName, visualizationName, detectorName);
   });
 
@@ -96,6 +107,14 @@ describe('Anomaly detection integration with vis augmenter', () => {
     associateDetectorFromVis(detectorName);
 
     ensureDetectorIsLinked(dashboardName, visualizationName, detectorName);
+
+    cy.visitDashboard(dashboardName);
+    setDateRangeTo7Days();
+    validateVisSnapshot(
+      visualizationName,
+      `${visualizationName}-with-associated-detector`
+    );
+
     unlinkDetectorFromVis(dashboardName, visualizationName, detectorName);
   });
 
@@ -105,6 +124,14 @@ describe('Anomaly detection integration with vis augmenter', () => {
     associateDetectorFromVis(detectorName);
 
     ensureDetectorIsLinked(dashboardName, visualizationName, detectorName);
+
+    cy.visitDashboard(dashboardName);
+    setDateRangeTo7Days();
+    validateVisSnapshot(
+      visualizationName,
+      `${visualizationName}-with-associated-detector-2`
+    );
+
     unlinkDetectorFromVis(dashboardName, visualizationName, detectorName);
   });
 

cypress/plugins/index.js

@@ -3,6 +3,10 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
+const {
+  addMatchImageSnapshotPlugin,
+} = require('cypress-image-snapshot/plugin');
+
 /// <reference types="cypress" />
 // ***********************************************************
 // This example plugins/index.js can be used to load plugins
@@ -24,4 +28,5 @@
 module.exports = (on, config) => {
   // `on` is used to hook into various events Cypress emits
   // `config` is the resolved Cypress config
+  addMatchImageSnapshotPlugin(on, config);
 };

cypress/support/commands.js

@@ -0,0 +1,12 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { addMatchImageSnapshotCommand } from 'cypress-image-snapshot/command';
+
+addMatchImageSnapshotCommand({
+  failureThreshold: 0.03, // threshold for entire image
+  failureThresholdType: 'percent', // percent of image or number of pixels
+  capture: 'viewport', // capture viewport in screenshot
+});

cypress/support/index.js

@@ -19,6 +19,7 @@
 // ***********************************************************
 
 // Import commands.js using ES2015 syntax:
+import './commands';
 import '../utils/commands';
 import '../utils/dashboards/commands';
 import '../utils/dashboards/datasource-management-dashboards-plugin/commands';

cypress/utils/dashboards/vis-augmenter/commands.js

@@ -17,6 +17,13 @@ Cypress.Commands.add('openVisContextMenu', { prevSubject: true }, (panel) =>
     .then(() => cy.get('.euiContextMenu'))
 );
 
+Cypress.Commands.add('closeVisContextMenu', { prevSubject: true }, (panel) =>
+  cy
+    .wrap(panel)
+    .find(`[data-test-subj="embeddablePanelContextMenuOpen"]`)
+    .click()
+);
+
 Cypress.Commands.add(
   'clickVisPanelMenuItem',
   { prevSubject: 'optional' },

cypress/utils/dashboards/vis-augmenter/helpers.js

@@ -320,3 +320,35 @@ export const filterByObjectType = (type) => {
     .click({ force: true });
   cy.wait(3000);
 };
+
+// Assumes runner is on dashboard
+export const validateVisSnapshot = (
+  visualizationName,
+  snapshotName,
+  maximizePanel = false
+) => {
+  if (maximizePanel) {
+    cy.getVisPanelByTitle(visualizationName)
+      .openVisContextMenu()
+      .clickVisPanelMenuItem('Maximize panel');
+    cy.wait(1000);
+    cy.get(`[data-title="${visualizationName}"]`).matchImageSnapshot(
+      snapshotName
+    );
+    cy.getVisPanelByTitle(visualizationName)
+      .openVisContextMenu()
+      .clickVisPanelMenuItem('Minimize');
+  } else {
+    cy.get(`[data-title="${visualizationName}"]`).matchImageSnapshot(
+      snapshotName
+    );
+  }
+};
+
+// Assumes runner is on dashboard
+export const setDateRangeTo7Days = () => {
+  cy.getElementByTestId('superDatePickerToggleQuickMenuButton').click();
+  cy.getElementByTestId('superDatePickerCommonlyUsed_Last_7 days').click();
+  cy.getElementByTestId('querySubmitButton').click();
+  cy.wait(2000);
+};