From 89fcf63ca0790858143a97c3bfb47460214afc7e Mon Sep 17 00:00:00 2001 From: Manasvini B Suryanarayana Date: Thu, 7 Mar 2024 18:09:29 -0800 Subject: [PATCH] Add functional test cypress workflow improvements and enable the workflow for in-house tests (#6061) Add cigroup matrix for parallel runs Signed-off-by: Manasvini B Suryanarayana --- .github/workflows/cypress_workflow.yml | 83 ++++++++++++++++--- ...kflow.yml => release_cypress_workflow.yml} | 64 ++++++++------ CHANGELOG.md | 2 + cypress/utils/commands.js | 15 ++++ package.json | 5 +- scripts/common/opensearch_service.sh | 1 + scripts/cypress_tests.sh | 6 +- 7 files changed, 136 insertions(+), 40 deletions(-) rename .github/workflows/{dashboards_cypress_workflow.yml => release_cypress_workflow.yml} (65%) create mode 100644 cypress/utils/commands.js diff --git a/.github/workflows/cypress_workflow.yml b/.github/workflows/cypress_workflow.yml index 14975e23b17a..e354be1415ce 100644 --- a/.github/workflows/cypress_workflow.yml +++ b/.github/workflows/cypress_workflow.yml @@ -47,7 +47,7 @@ jobs: strategy: fail-fast: false matrix: - group: [1, 2, 3, 4, 5, 6, 7, 8, 9] + group: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] container: image: docker://opensearchstaging/ci-runner:ci-runner-rockylinux8-opensearch-dashboards-integtest-v2 options: --user 1001 @@ -101,7 +101,7 @@ jobs: - name: Build plugins run: node scripts/build_opensearch_dashboards_platform_plugins --no-examples --workers 12 - - name: Checkout + - name: Checkout FT repo uses: actions/checkout@v2 with: path: ${{ env.FTR_PATH }} @@ -113,8 +113,9 @@ jobs: run: | echo "SPEC=${{ inputs.specs }}" >> $GITHUB_ENV + # Setup spec files for existing Functional Test repo cypress tests - name: Setup spec files - if: ${{ inputs.specs == '' }} + if: ${{ inputs.specs == '' && matrix.group < 10 }} working-directory: ${{ env.FTR_PATH }} shell: bash run: | @@ -126,11 +127,13 @@ jobs: echo "SPEC=${FORMATTED_SPEC}" >> $GITHUB_ENV - name: Get Cypress version + if: ${{ matrix.group < 10 }} id: cypress_version run: | echo "name=cypress_version::$(cat ./${{ env.FTR_PATH }}/package.json | jq '.devDependencies.cypress' | tr -d '"')" >> $GITHUB_OUTPUT - + - name: Cache Cypress + if: ${{ matrix.group < 10 }} id: cache-cypress uses: actions/cache@v1 with: @@ -141,36 +144,92 @@ jobs: - run: npx cypress cache list - run: npx cypress cache path - - name: Run tests + # Setup spec files for Dashboards in-house cypress tests + - name: Setup spec files for Dashboards tests + if: ${{ inputs.specs == '' && matrix.group > 9 }} + shell: bash + run: | + IFS="," read -a SPEC_ARRAY <<< $(yarn --silent osd:ciGroup${{ matrix.group }}) + DASHBOARDS_SPEC='' + for i in "${SPEC_ARRAY[@]}"; do + DASHBOARDS_SPEC+="cypress/integration/core_opensearch_dashboards/${i}," + done + echo "DASHBOARDS_SPEC=${DASHBOARDS_SPEC}" >> $GITHUB_ENV + + # Run FT repo tests + - name: Run FT repo tests + if: ${{ matrix.group < 10 }} uses: cypress-io/github-action@v2 with: working-directory: ${{ env.FTR_PATH }} start: ${{ env.OPENSEARCH_SNAPSHOT_CMD }}, ${{ env.START_CMD }} wait-on: 'http://localhost:9200, http://localhost:5601' command: yarn cypress:run-without-security --browser ${{ env.CYPRESS_BROWSER }} --spec ${{ env.SPEC }} + + # Clear Cypress Cache before running Dashboards tests + - name: Clear Cypress Cache + if: ${{ matrix.group > 9 }} + run: npx cypress cache clear + + # Run Dashboards Cypress tests within the source repo + - name: Run Dashboards Cypress tests + if: ${{ matrix.group > 9 }} + uses: cypress-io/github-action@v6 + with: + install-command: npx cypress install --force + start: node scripts/opensearch snapshot -E cluster.routing.allocation.disk.threshold_enabled=false, node scripts/opensearch_dashboards --dev --no-base-path --no-watch --savedObjects.maxImportPayloadBytes=10485760 --server.maxPayloadBytes=1759977 --logging.json=false --data.search.aggs.shardDelay.enabled=true + wait-on: 'http://localhost:9200, http://localhost:5601' + command: yarn cypress:run-without-security --browser ${{ env.CYPRESS_BROWSER }} --spec ${{ env.DASHBOARDS_SPEC }} # Screenshots are only captured on failure, will change this once we do visual regression tests - - uses: actions/upload-artifact@v3 - if: failure() + - name: Upload FT repo screenshots + uses: actions/upload-artifact@v3 + if: failure() && (matrix.group < 10) with: name: ftr-cypress-screenshots path: ${{ env.FTR_PATH }}/cypress/screenshots retention-days: 1 - - - uses: actions/upload-artifact@v3 - if: always() + + - name: Upload FT repo videos + uses: actions/upload-artifact@v3 + if: always() && (matrix.group < 10) with: name: ftr-cypress-videos path: ${{ env.FTR_PATH }}/cypress/videos retention-days: 1 - - uses: actions/upload-artifact@v3 - if: always() + - name: Upload FT repo results + uses: actions/upload-artifact@v3 + if: always() && (matrix.group < 10) with: name: ftr-cypress-results path: ${{ env.FTR_PATH }}/cypress/results retention-days: 1 + - name: Upload Dashboards screenshots + if: failure() && (matrix.group > 9) + uses: actions/upload-artifact@v3 + with: + name: dashboards-cypress-screenshots + path: cypress/screenshots + retention-days: 1 + + - name: Upload Dashboards repo videos + uses: actions/upload-artifact@v3 + if: always() && (matrix.group > 9) + with: + name: dashboards-cypress-videos + path: cypress/videos + retention-days: 1 + + - name: Upload Dashboards repo results + uses: actions/upload-artifact@v3 + if: always() && (matrix.group > 9) + with: + name: dashboards-cypress-results + path: cypress/results + retention-days: 1 + add-comment: needs: [cypress-tests] if: ${{ always() && github.event_name == 'workflow_dispatch' && inputs.pr_number != '' }} diff --git a/.github/workflows/dashboards_cypress_workflow.yml b/.github/workflows/release_cypress_workflow.yml similarity index 65% rename from .github/workflows/dashboards_cypress_workflow.yml rename to .github/workflows/release_cypress_workflow.yml index ce578970b477..f58757b23f9f 100644 --- a/.github/workflows/dashboards_cypress_workflow.yml +++ b/.github/workflows/release_cypress_workflow.yml @@ -1,5 +1,5 @@ name: Orchestrator cypress workflow -run-name: dashboards_cypress_workflow ${{ inputs.UNIQUE_ID != '' && inputs.UNIQUE_ID || '' }} # Unique id number appended to the workflow run-name to reference the run within the orchestrator. +run-name: release_cypress_workflow ${{ inputs.UNIQUE_ID != '' && inputs.UNIQUE_ID || '' }} # Unique id number appended to the workflow run-name to reference the run within the orchestrator. # Trigger on dispatch event sent from FT repo orchestrator on: workflow_dispatch: @@ -41,27 +41,38 @@ env: TEST_REPO: ${{ inputs.test_repo != '' && inputs.test_repo || github.repository }} TEST_BRANCH: "${{ inputs.test_branch != '' && inputs.test_branch || github.base_ref }}" OSD_PATH: 'osd' - CYPRESS_BROWSER: 'chromium' + CYPRESS_BROWSER: 'electron' JOB_ID: ${{ inputs.UNIQUE_ID}} OPENSEARCH: ${{ inputs.OS_URL != '' && inputs.OS_URL || 'https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/$VERSION/latest/linux/x64/tar/dist/opensearch/opensearch-$VERSION-linux-x64.tar.gz' }} DASHBOARDS: ${{ inputs.OSD_URL != '' && inputs.OSD_URL || 'https://ci.opensearch.org/ci/dbc/distribution-build-opensearch-dashboards/$VERSION/latest/linux/x64/tar/dist/opensearch-dashboards/opensearch-dashboards-$VERSION-linux-x64.tar.gz' }} OPENSEARCH_DIR: 'cypress/opensearch' DASHBOARDS_DIR: 'cypress/opensearch-dashboards' SECURITY_ENABLED: ${{ inputs.SECURITY_ENABLED != '' && inputs.SECURITY_ENABLED || 'false' }} - SPEC: 'cypress/integration/core_opensearch_dashboards/*' jobs: + Get-CI-Image-Tag: + uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main + with: + product: opensearch-dashboards + cypress-tests: + needs: Get-CI-Image-Tag runs-on: ubuntu-latest + strategy: + matrix: + spec_group: [10, 11] # Add more spec files as needed container: - image: docker://opensearchstaging/ci-runner:ci-runner-rockylinux8-opensearch-dashboards-integtest-v2 - options: --user 1001 + # using the same image which is used by opensearch-build team to build the OpenSearch Distribution + # this image tag is subject to change as more dependencies and updates will arrive over time + image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }} + # need to switch to root so that github actions can install runner binary on container without permission issues. + options: --user root env: # prevents extra Cypress installation progress messages CI: 1 # avoid warnings like "tput: No value for $TERM and no -T specified" TERM: xterm - name: Run cypress tests ${{ inputs.UNIQUE_ID}} + name: Run cypress tests (osd:ciGroup${{ matrix.spec_group }}) ${{ inputs.UNIQUE_ID}} steps: - name: Checkout code uses: actions/checkout@v2 @@ -82,21 +93,24 @@ jobs: npm i -g yarn@1.22.10 yarn config set network-timeout 1000000 -g + - name: Setup spec files + if: ${{ inputs.specs == '' }} + shell: bash + run: | + cd ${{ env.OSD_PATH }} + IFS="," read -a SPEC_ARRAY <<< $(yarn --silent osd:ciGroup${{ matrix.spec_group }}) + FORMATTED_SPEC='' + for i in "${SPEC_ARRAY[@]}"; do + FORMATTED_SPEC+="cypress/integration/core_opensearch_dashboards/${i}," + done + SPEC=${FORMATTED_SPEC} # Set SPEC to FORMATTED_SPEC + echo "SPEC=${FORMATTED_SPEC}" >> $GITHUB_ENV + echo "SPEC found: $SPEC" + - name: Get Cypress version id: cypress_version run: | echo "name=cypress_version::$(cat ./${{ env.OSD_PATH }}/package.json | jq '.devDependencies.cypress' | tr -d '"')" >> $GITHUB_OUTPUT - - - name: Cache Cypress - id: cache-cypress - uses: actions/cache@v1 - with: - path: ~/.cache/Cypress - key: cypress-cache-v2-${{ runner.os }}-${{ hashFiles('**/package.json') }} - env: - CYPRESS_INSTALL_BINARY: ${{ steps.cypress_version.outputs.cypress_version }} - - run: npx cypress cache list - - run: npx cypress cache path - name: Get package version (Linux) run: | @@ -106,16 +120,18 @@ jobs: - name: Run bootstrap run: | cd ${{ env.OSD_PATH }} - yarn osd bootstrap + chown -R 1000:1000 `pwd` + su `id -un 1000` -c "source $NVM_DIR/nvm.sh && nvm use && node -v && yarn -v && + yarn osd bootstrap" - - name: Download and extract Opensearch artifacts + - name: Download and extract OpenSearch artifacts run: | CWD=$(pwd) mkdir -p $CWD/${{ env.OPENSEARCH_DIR }} source ${{ env.OSD_PATH }}/scripts/common/utils.sh open_artifact $CWD/${{ env.OPENSEARCH_DIR }} ${{ env.OPENSEARCH }} - - name: Download and extract Opensearch Dashboards artifacts + - name: Download and extract OpenSearch Dashboards artifacts run: | CWD=$(pwd) mkdir -p $CWD/${{ env.DASHBOARDS_DIR }} @@ -124,20 +140,20 @@ jobs: - name: Run Cypress tests run: | - source ${{ env.OSD_PATH }}/scripts/cypress_tests.sh - run_dashboards_cypress_tests + chown -R 1000:1000 `pwd` + su `id -un 1000` -c "source ${{ env.OSD_PATH }}/scripts/cypress_tests.sh && run_dashboards_cypress_tests" # Screenshots are only captured on failures - uses: actions/upload-artifact@v3 if: failure() with: - name: osd-cypress-screenshots + name: release-osd-cypress-screenshots path: ${{ env.OSD_PATH }}/cypress/screenshots retention-days: 1 - uses: actions/upload-artifact@v3 if: always() with: - name: osd-cypress-videos + name: release-osd-cypress-videos path: ${{ env.OSD_PATH }}/cypress/videos retention-days: 1 diff --git a/CHANGELOG.md b/CHANGELOG.md index 5dbfcbd705fc..2235557a18e9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -54,6 +54,8 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) ### 🔩 Tests +- Add functional test cypress workflow improvements and enable the workflow for in-house Dashboards tests ([#6061](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/6061)) + ## [2.12.0 - 2024-02-20](https://github.com/opensearch-project/OpenSearch-Dashboards/releases/tag/2.12.0) ### 💥 Breaking Changes diff --git a/cypress/utils/commands.js b/cypress/utils/commands.js new file mode 100644 index 000000000000..56a1fd0cff0e --- /dev/null +++ b/cypress/utils/commands.js @@ -0,0 +1,15 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +// --- Typed commands -- + +Cypress.Commands.add('getElementByTestId', (testId, options = {}) => { + return cy.get(`[data-test-subj="${testId}"]`, options); +}); + +Cypress.Commands.add('getElementsByTestIds', (testIds, options = {}) => { + const selectors = [testIds].flat(Infinity).map((testId) => `[data-test-subj="${testId}"]`); + return cy.get(selectors.join(','), options); +}); diff --git a/package.json b/package.json index 6305cbed15bf..b1bb98255e81 100644 --- a/package.json +++ b/package.json @@ -79,8 +79,9 @@ "spec_to_console": "scripts/use_node scripts/spec_to_console", "pkg-version": "scripts/use_node -e \"console.log(require('./package.json').version)\"", "cypress:run-without-security": "env TZ=America/Los_Angeles NO_COLOR=1 cypress run --headless --env SECURITY_ENABLED=false", - "cypress:run-with-security": "env TZ=America/Los_Angeles NO_COLOR=1 cypress run --headless --env SECURITY_ENABLED=true,openSearchUrl=https://localhost:9200,WAIT_FOR_LOADER_BUFFER_MS=500" - + "cypress:run-with-security": "env TZ=America/Los_Angeles NO_COLOR=1 cypress run --headless --env SECURITY_ENABLED=true,openSearchUrl=https://localhost:9200,WAIT_FOR_LOADER_BUFFER_MS=500", + "osd:ciGroup10": "echo \"dashboard_sanity_test_spec.js\"", + "osd:ciGroup11": "echo \"apps/vis_builder/*.js\"" }, "repository": { "type": "git", diff --git a/scripts/common/opensearch_service.sh b/scripts/common/opensearch_service.sh index 09f3b5d50c8c..120caee12d26 100755 --- a/scripts/common/opensearch_service.sh +++ b/scripts/common/opensearch_service.sh @@ -9,6 +9,7 @@ function setup_opensearch() { cd "$OPENSEARCH_DIR" echo "network.host: 0.0.0.0" >> config/opensearch.yml echo "discovery.type: single-node" >> config/opensearch.yml + echo "cluster.routing.allocation.disk.threshold_enabled: false" >> config/opensearch.yml [ $SECURITY_ENABLED == "false" ] && [ -d "plugins/opensearch-security" ] && echo "plugins.security.disabled: true" >> config/opensearch.yml # Required for IM [ -d "plugins/opensearch-index-management" ] && echo "path.repo: [/tmp]" >> config/opensearch.yml diff --git a/scripts/cypress_tests.sh b/scripts/cypress_tests.sh index e4c2cbfc9352..93e58664f438 100644 --- a/scripts/cypress_tests.sh +++ b/scripts/cypress_tests.sh @@ -67,10 +67,12 @@ function check_status() { # Starts OpenSearch Dashboards and run tests in the cypress folder function run_dashboards_cypress_tests() { + run_opensearch + cd $CWD echo "[ OpenSearch Dashboards setup before it starts... ]" setup_dashboards >> /dev/null 2>&1 & sleep 100 - cd "$DASHBOARDS_DIR" + cd $CWD/"$DASHBOARDS_DIR" if [ -x "./bin/opensearch-dashboards" ]; then spawn_process_and_save_PID "./bin/opensearch-dashboards &" else @@ -79,11 +81,11 @@ function run_dashboards_cypress_tests() { fi check_status $DASHBOARDS_URL $DASHBOARDS_MSG # Run cypress tests + cd "$CWD"/osd run_cypress } function run_cypress() { - cd "$CWD"/osd echo "SPEC found: $SPEC" if [ $SECURITY_ENABLED = "true" ]; then echo "run security enabled tests"