[CI][BWC] add BWC tests to github actions

.github/workflows/build_and_test_workflow.yml

@@ -23,9 +23,11 @@ env:
 jobs:
   build-lint-test:
     runs-on: ubuntu-latest
+    container:
+      image: docker://opensearchstaging/ci-runner:ci-runner-rockylinux8-opensearch-dashboards-integtest-v2
+      options: --user 1001
     name: Build and Verify
     steps:
-
       - name: Checkout code
         uses: actions/checkout@v2
 
@@ -67,6 +69,9 @@ jobs:
 
   functional-tests:
     runs-on: ubuntu-latest
+    container:
+      image: docker://opensearchstaging/ci-runner:ci-runner-rockylinux8-opensearch-dashboards-integtest-v2
+      options: --user 1001
     name: Run functional tests
     strategy: 
       matrix:
@@ -88,9 +93,9 @@ jobs:
           npm uninstall -g yarn
           npm i -g yarn@1.22.10
 
-      # github virtual env is the latest chrome
+      # image has the latest chrome v99
       - name: Setup chromedriver
-        run: yarn add --dev chromedriver@100.0.0
+        run: yarn add --dev chromedriver@99.0.0
 
       - name: Run bootstrap
         run: yarn osd bootstrap
@@ -109,6 +114,9 @@ jobs:
 
   build-min-artifact-tests:
     runs-on: ubuntu-latest
+    container:
+      image: docker://opensearchstaging/ci-runner:ci-runner-rockylinux8-opensearch-dashboards-integtest-v2
+      options: --user 1001
     name: Build min release artifacts
     defaults:
       run:
@@ -158,6 +166,71 @@ jobs:
       - uses: actions/upload-artifact@v3
         if: success()
         with:
-          name: ${{ matrix.name }}
+          name: ${{ matrix.suffix }}-${{ env.VERSION }}
           path: ./artifacts/target/${{ env.ARTIFACT_BUILD_NAME }}
-          retention-days: 1
+          retention-days: 1
+
+  bwc-tests:
+    needs: [ build-min-artifact-tests ]
+    runs-on: ubuntu-latest
+    container:
+      image: docker://opensearchstaging/ci-runner:ci-runner-rockylinux8-opensearch-dashboards-integtest-v2
+      options: --user 1001
+    name: Run backwards compatibility tests
+    defaults:
+      run:
+        working-directory: ./artifacts
+    strategy: 
+      matrix:
+        version: [ osd-2.0.0 ]
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+        with:
+          path: ./artifacts
+
+      - run: echo Running backwards compatibility tests for version ${{ matrix.version }}
+      - run: echo [NOTE] These tests will be ran using Linux x64 release builds without security
+
+      - name: Setup Node
+        uses: actions/setup-node@v2
+        with:
+          node-version-file: "./artifacts/.nvmrc"
+          registry-url: 'https://registry.npmjs.org'
+
+      - name: Setup Yarn
+        run: |
+          npm uninstall -g yarn
+          npm i -g yarn@1.22.10
+
+      - name: Get package version
+        run: |
+          echo "VERSION=$(yarn --silent pkg-version)" >> $GITHUB_ENV
+
+      - name: Set OpenSearch URL
+        run: |
+          echo "OPENSEARCH_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/${{ env.VERSION }}/latest/linux/x64/tar/dist/opensearch/opensearch-${{ env.VERSION }}-linux-x64.tar.gz" >> $GITHUB_ENV
+
+      - name: Verify if OpenSearch is available for version
+        id: verify-opensearch-exists
+        run: |
+          if curl -I -L ${{ env.OPENSEARCH_URL }}; then
+            echo "::set-output name=version-exists::true"
+          fi
+
+      - name: Skipping tests
+        if: steps.verify-opensearch-exists.outputs.version-exists != 'true'
+        run: echo Tests were skipped because an OpenSearch release build does not exist for this version yet!
+
+      - name: Download OpenSearch Dashboards
+        uses: actions/download-artifact@v3
+        id: download
+        with:
+          name: linux-x64-${{ env.VERSION }}
+          path: ./artifacts/
+        if: steps.verify-opensearch-exists.outputs.version-exists == 'true'
+
+      - name: Run tests
+        if: steps.verify-opensearch-exists.outputs.version-exists == 'true'
+        run: |
+          ./bwctest.sh -s false -o ${{ env.OPENSEARCH_URL }} -d ${{ steps.download.outputs.download-path }}/opensearch-dashboards-${{ env.VERSION }}-linux-x64.tar.gz

TESTING.md

@@ -67,6 +67,12 @@ To run specific versions' backwards compatibility tests, pass the versions to th
 
 `yarn test:bwc -o [test path to opensearch.tar.gz] -d [test path to opensearch-dashboards.tar.gz] -v "[test versions]"`
 
+To generate test data that will be utilized for backwards compatibility tests:
+
+`yarn test:bwc -o [test path to opensearch.tar.gz] -d [test path to opensearch-dashboards.tar.gz] -g true`
+
+This will create an archive of the data based on the OpenSearch Dashboards version you have provided. For example, if a tarball of 2.0.0 was passed then an `osd-2.0.0.zip` will be created. This command is intended to be executed when needed per a version. For example, when end-users cannot migrate directly from `vPrevious` to `vNext`. If `osd-vCurrent.zip` does not exist, then this command be ran and the output sourced controlled for future use.
+
 ### Additional checks
 Make sure you run lint checker before submitting a pull request. To run lint checker:
 `node scripts/precommit_hook.js --fix`

bwctest.sh

@@ -13,7 +13,7 @@
 
 set -e
 
-DEFAULT_VERSIONS="osd-1.1.0,odfe-1.13.2,odfe-1.0.2"
+DEFAULT_VERSIONS="osd-2.0.0,"
 
 function usage() {
     echo ""
@@ -75,6 +75,7 @@ done
 [ -z "$SECURITY_ENABLED" ] && SECURITY_ENABLED="false"
 [ -z "$CREDENTIAL" ] && CREDENTIAL="admin:admin"
 [ -z "$CI" ] && CI=1
+[ -z "$BWC_VERSIONS" ] && BWC_VERSIONS=$DEFAULT_VERSIONS
 
 # If no OpenSearch build was passed then this constructs the version
 if [ -z "$OPENSEARCH" ]; then
@@ -96,4 +97,4 @@ if [ -z "$OPENSEARCH" ]; then
     OPENSEARCH="https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/$VERSION/latest/$PLATFORM/$ARCH/tar/dist/opensearch/opensearch-$VERSION-$PLATFORM-$ARCH.tar.gz"
 fi
 
-source scripts/bwctest_osd.sh -b $BIND_ADDRESS -p $BIND_PORT -s $SECURITY_ENABLED -c $CREDENTIAL -o $OPENSEARCH -d $DASHBOARDS -v $DEFAULT_VERSIONS
+source scripts/bwctest_osd.sh -b $BIND_ADDRESS -p $BIND_PORT -s $SECURITY_ENABLED -c $CREDENTIAL -o $OPENSEARCH -d $DASHBOARDS -v $BWC_VERSIONS

cypress/integration/with-security/check_loaded_data.js

@@ -47,7 +47,9 @@ describe('check previously loaded data', () => {
         .get('[data-test-subj="dashboardListingTitleLink-[Flights]-Global-Flight-Dashboard"]')
         .click();
       commonUI.removeAllFilters();
-      commonUI.setDateRange('Dec 1, 2021 @ 00:00:00.000', 'Nov 1, 2021 @ 00:00:00.000');
+      commonUI.setDateRange(
+        `Dec 31, ${new Date().getFullYear()} @ 00:00:00.000', 'Nov 1, 2016 @ 00:00:00.000`
+      );
     });
 
     it('Global Flight Dashboard is loaded and funtions correctly', () => {
@@ -77,7 +79,9 @@ describe('check previously loaded data', () => {
         .get('[data-test-subj="dashboardListingTitleLink-[eCommerce]-Revenue-Dashboard"]')
         .click();
       commonUI.removeAllFilters();
-      commonUI.setDateRange('Nov 1, 2021 @ 00:00:00.000', 'Nov 1, 2016 @ 00:00:00.000');
+      commonUI.setDateRange(
+        `Dec 31, ${new Date().getFullYear()} @ 00:00:00.000', 'Nov 1, 2016 @ 00:00:00.000`
+      );
     });
 
     it('eCommerce Revenue Dashboard is loaded and functions correctly', () => {

cypress/integration/with-security/helpers/generate_data.js

@@ -0,0 +1,86 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import {
+  MiscUtils,
+  LoginPage,
+} from '@opensearch-dashboards-test/opensearch-dashboards-test-library';
+
+const miscUtils = new MiscUtils(cy);
+const loginPage = new LoginPage(cy);
+
+describe('Generating BWC test data with security', () => {
+  beforeEach(() => {
+    miscUtils.visitPage('app/management/opensearch-dashboards/settings');
+    loginPage.enterUserName('admin');
+    loginPage.enterPassword('admin');
+    loginPage.submit();
+  });
+
+  afterEach(() => {
+    cy.clearCookies();
+  });
+
+  it('tenant-switch-modal page should show and be clicked', () => {
+    cy.get('[data-test-subj="tenant-switch-modal"]');
+    cy.get('[data-test-subj="confirm"]').click();
+  });
+
+  it('adds advanced settings', () => {
+    miscUtils.visitPage('app/management/opensearch-dashboards/settings');
+    cy.get('[data-test-subj="advancedSetting-editField-theme:darkMode"]').click();
+    cy.get('[data-test-subj="advancedSetting-editField-timeline:default_columns"]').type(
+      '{selectAll}4'
+    );
+    cy.get('[data-test-subj="advancedSetting-editField-timeline:max_buckets"]').type(
+      '{selectAll}4'
+    );
+    cy.get('[data-test-subj="advancedSetting-editField-defaultRoute"]')
+      .clear()
+      .type('/app/opensearch_dashboards_overview#/');
+    cy.get('[data-test-subj="advancedSetting-saveButton"]').click({ force: true });
+    cy.reload();
+  });
+
+  it('adds sample data', () => {
+    miscUtils.addSampleData();
+  });
+
+  it('adds filters and queries', () => {
+    miscUtils.visitPage('app/dashboards#');
+    cy.get('[data-test-subj="dashboardListingTitleLink-[Logs]-Web-Traffic"]').click();
+    cy.get('[data-test-subj="queryInput"]').clear().type('resp=200');
+    cy.get('[data-test-subj="addFilter"]').click();
+    cy.get('[data-test-subj="filterFieldSuggestionList"]')
+      .find('[data-test-subj="comboBoxInput"]')
+      .type('machine.os{downArrow}{enter}');
+    cy.get('[data-test-subj="filterOperatorList"]')
+      .find('[data-test-subj="comboBoxInput"]')
+      .type('{downArrow}{enter}');
+    cy.get('[data-test-subj="filterParams"]')
+      .find('input.euiFieldText[placeholder="Enter a value"]')
+      .type('osx');
+    cy.get('[data-test-subj="createCustomLabel"]').click();
+    cy.get('[class="globalFilterItem__editorForm"]').find('input').last().type('osx filter');
+    cy.get('[data-test-subj="saveFilter"]').click();
+    cy.get('[data-test-subj="saved-query-management-popover-button"]').click();
+    cy.get('[data-test-subj="saved-query-management-popover"]')
+      .find('[data-test-subj="saved-query-management-save-button"]')
+      .click();
+    cy.get('[data-test-subj="saveQueryFormTitle"]').type('test-query');
+    cy.get('[data-test-subj="savedQueryFormSaveButton"]').click();
+  });
+
+  it('adds Timeline visualization', () => {
+    miscUtils.visitPage('app/visualize#');
+    cy.get('[data-test-subj="visualizationLandingPage"]')
+      .find('[data-test-subj="newItemButton"]')
+      .click();
+    cy.get('[data-test-subj="visType-timelion"]').click();
+    cy.get('[data-test-subj="visualizeSaveButton"]').click();
+    cy.get('[data-test-subj="savedObjectTitle"]').type('test-timeline');
+    cy.get('[data-test-subj="confirmSaveSavedObjectButton"]').click();
+  });
+});

cypress/integration/without-security/check_loaded_data.js

@@ -38,7 +38,9 @@ describe('check previously loaded data', () => {
         .get('[data-test-subj="dashboardListingTitleLink-[Flights]-Global-Flight-Dashboard"]')
         .click();
       commonUI.removeAllFilters();
-      commonUI.setDateRange('Dec 1, 2021 @ 00:00:00.000', 'Nov 1, 2021 @ 00:00:00.000');
+      commonUI.setDateRange(
+        `Dec 31, ${new Date().getFullYear()} @ 00:00:00.000', 'Nov 1, 2016 @ 00:00:00.000`
+      );
     });
 
     it('Global Flight Dashboard is loaded when clicked', () => {
@@ -71,7 +73,9 @@ describe('check previously loaded data', () => {
         .get('[data-test-subj="dashboardListingTitleLink-[eCommerce]-Revenue-Dashboard"]')
         .click();
       commonUI.removeAllFilters();
-      commonUI.setDateRange('Nov 1, 2021 @ 00:00:00.000', 'Nov 1, 2016 @ 00:00:00.000');
+      commonUI.setDateRange(
+        `Dec 31, ${new Date().getFullYear()} @ 00:00:00.000', 'Nov 1, 2016 @ 00:00:00.000`
+      );
     });
 
     it('eCommerce Revenue Dashboard is loaded when clicked', () => {

cypress/integration/without-security/helpers/generate_data.js

@@ -0,0 +1,69 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { MiscUtils } from '@opensearch-dashboards-test/opensearch-dashboards-test-library';
+
+const miscUtils = new MiscUtils(cy);
+
+describe('Generating BWC test data without security', () => {
+  before(() => {
+    miscUtils.visitPage('app');
+  });
+  it('adds advanced settings', () => {
+    miscUtils.visitPage('app/management/opensearch-dashboards/settings');
+    cy.get('[data-test-subj="advancedSetting-editField-theme:darkMode"]').click();
+    cy.get('[data-test-subj="advancedSetting-editField-timeline:default_columns"]').type(
+      '{selectAll}4'
+    );
+    cy.get('[data-test-subj="advancedSetting-editField-timeline:max_buckets"]').type(
+      '{selectAll}4'
+    );
+    cy.get('[data-test-subj="advancedSetting-editField-defaultRoute"]')
+      .clear()
+      .type('/app/opensearch_dashboards_overview#/');
+    cy.get('[data-test-subj="advancedSetting-saveButton"]').click({ force: true });
+    cy.reload();
+  });
+
+  it('adds sample data', () => {
+    miscUtils.addSampleData();
+  });
+
+  it('adds filters and queries', () => {
+    miscUtils.visitPage('app/dashboards#');
+    cy.get('[data-test-subj="dashboardListingTitleLink-[Logs]-Web-Traffic"]').click();
+    cy.get('[data-test-subj="queryInput"]').clear().type('resp=200');
+    cy.get('[data-test-subj="addFilter"]').click();
+    cy.get('[data-test-subj="filterFieldSuggestionList"]')
+      .find('[data-test-subj="comboBoxInput"]')
+      .type('machine.os{downArrow}{enter}');
+    cy.get('[data-test-subj="filterOperatorList"]')
+      .find('[data-test-subj="comboBoxInput"]')
+      .type('{downArrow}{enter}');
+    cy.get('[data-test-subj="filterParams"]')
+      .find('input.euiFieldText[placeholder="Enter a value"]')
+      .type('osx');
+    cy.get('[data-test-subj="createCustomLabel"]').click();
+    cy.get('[class="globalFilterItem__editorForm"]').find('input').last().type('osx filter');
+    cy.get('[data-test-subj="saveFilter"]').click();
+    cy.get('[data-test-subj="saved-query-management-popover-button"]').click();
+    cy.get('[data-test-subj="saved-query-management-popover"]')
+      .find('[data-test-subj="saved-query-management-save-button"]')
+      .click();
+    cy.get('[data-test-subj="saveQueryFormTitle"]').type('test-query');
+    cy.get('[data-test-subj="savedQueryFormSaveButton"]').click();
+  });
+
+  it('adds Timeline visualization', () => {
+    miscUtils.visitPage('app/visualize#');
+    cy.get('[data-test-subj="visualizationLandingPage"]')
+      .find('[data-test-subj="newItemButton"]')
+      .click();
+    cy.get('[data-test-subj="visType-timelion"]').click();
+    cy.get('[data-test-subj="visualizeSaveButton"]').click();
+    cy.get('[data-test-subj="savedObjectTitle"]').type('test-timeline');
+    cy.get('[data-test-subj="confirmSaveSavedObjectButton"]').click();
+  });
+});

scripts/bwc/generate_test_data.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+# Copyright OpenSearch Contributors
+# SPDX-License-Identifier: Apache-2.0
+
+set -e
+
+function run_generate_data_spec() {
+  echo "[ Generating test data ]"
+  cd "$TEST_DIR"
+  [ "$CI" == '1' ] && cypress_args="--browser chromium" || cypress_args=""
+  SPEC_FILE="$TEST_DIR/cypress/integration/$DASHBOARDS_TYPE/helpers/generate_data.js"
+  env NO_COLOR=1 npx cypress run $cypress_args --headless --spec $SPEC_FILE || true
+}
+
+function archive_data() {
+  echo "[ Archiving test data. Beep boo-boo, boo-boo bop ]"
+  PACKAGE_VERSION=$(get_dashboards_package_version)
+  cd "$OPENSEARCH_DIR" && tar -zcvf "osd-$PACKAGE_VERSION.tar.gz" data
+  cp "osd-$PACKAGE_VERSION.tar.gz" "$CWD/cypress/test-data/$DASHBOARDS_TYPE"
+  echo "[ Archive complete. Location: $CWD/cypress/test-data/$DASHBOARDS_TYPE/osd-$PACKAGE_VERSION.tar.gz' ]"
+}

scripts/bwc/opensearch_dashboards_service.sh

@@ -27,6 +27,6 @@ function run_dashboards() {
 function check_dashboards_status {
   echo "Checking the OpenSearch Dashboards..."
   cd "$DIR"
-  check_status $DASHBOARDS_PATH "$DASHBOARDS_MSG" $DASHBOARDS_URL "" >> /dev/null 2>&1
+  check_status $DASHBOARDS_PATH "$DASHBOARDS_MSG" $DASHBOARDS_URL "$OPENSEARCH_ARGS" >> /dev/null 2>&1
   echo "OpenSearch Dashboards is up!"
 }