diff --git a/.github/workflows/cypress_workflow.yml b/.github/workflows/cypress_workflow.yml index cab511ddc551..5a3524d97c44 100644 --- a/.github/workflows/cypress_workflow.yml +++ b/.github/workflows/cypress_workflow.yml @@ -47,7 +47,7 @@ jobs: strategy: fail-fast: false matrix: - group: [1, 2, 3, 4, 5, 6, 7, 8, 9] + group: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] container: image: docker://opensearchstaging/ci-runner:ci-runner-rockylinux8-opensearch-dashboards-integtest-v2 options: --user 1001 @@ -101,7 +101,7 @@ jobs: - name: Build plugins run: node scripts/build_opensearch_dashboards_platform_plugins --no-examples --workers 12 - - name: Checkout + - name: Checkout FT repo uses: actions/checkout@v2 with: path: ${{ env.FTR_PATH }} @@ -113,8 +113,9 @@ jobs: run: | echo "SPEC=${{ inputs.specs }}" >> $GITHUB_ENV + # Setup spec files for existing Functional Test repo cypress tests - name: Setup spec files - if: ${{ inputs.specs == '' }} + if: ${{ inputs.specs == '' && matrix.group < 10 }} working-directory: ${{ env.FTR_PATH }} shell: bash run: | @@ -126,11 +127,13 @@ jobs: echo "SPEC=${FORMATTED_SPEC}" >> $GITHUB_ENV - name: Get Cypress version + if: ${{ matrix.group < 10 }} id: cypress_version run: | echo "name=cypress_version::$(cat ./${{ env.FTR_PATH }}/package.json | jq '.devDependencies.cypress' | tr -d '"')" >> $GITHUB_OUTPUT - + - name: Cache Cypress + if: ${{ matrix.group < 10 }} id: cache-cypress uses: actions/cache@v1 with: @@ -141,36 +144,92 @@ jobs: - run: npx cypress cache list - run: npx cypress cache path - - name: Run tests + # Setup spec files for Dashboards in-house cypress tests + - name: Setup spec files for Dashboards tests + if: ${{ inputs.specs == '' && matrix.group > 9 }} + shell: bash + run: | + IFS="," read -a SPEC_ARRAY <<< $(yarn --silent osd:ciGroup${{ matrix.group }}) + DASHBOARDS_SPEC='' + for i in "${SPEC_ARRAY[@]}"; do + DASHBOARDS_SPEC+="cypress/integration/core_opensearch_dashboards/${i}," + done + echo "DASHBOARDS_SPEC=${DASHBOARDS_SPEC}" >> $GITHUB_ENV + + # Run FT repo tests + - name: Run FT repo tests + if: ${{ matrix.group < 10 }} uses: cypress-io/github-action@v2 with: working-directory: ${{ env.FTR_PATH }} start: ${{ env.OPENSEARCH_SNAPSHOT_CMD }}, ${{ env.START_CMD }} wait-on: 'http://localhost:9200, http://localhost:5601' command: yarn cypress:run-without-security --browser ${{ env.CYPRESS_BROWSER }} --spec ${{ env.SPEC }} + + # Clear Cypress Cache before running Dashboards tests + - name: Clear Cypress Cache + if: ${{ matrix.group > 9 }} + run: npx cypress cache clear + + # Run Dashboards Cypress tests within the source repo + - name: Run Dashboards Cypress tests + if: ${{ matrix.group > 9 }} + uses: cypress-io/github-action@v6 + with: + install-command: npx cypress install --force + start: node scripts/opensearch snapshot -E cluster.routing.allocation.disk.threshold_enabled=false, node scripts/opensearch_dashboards --dev --no-base-path --no-watch --savedObjects.maxImportPayloadBytes=10485760 --server.maxPayloadBytes=1759977 --logging.json=false --data.search.aggs.shardDelay.enabled=true + wait-on: 'http://localhost:9200, http://localhost:5601' + command: yarn cypress:run-without-security --browser ${{ env.CYPRESS_BROWSER }} --spec ${{ env.DASHBOARDS_SPEC }} # Screenshots are only captured on failure, will change this once we do visual regression tests - - uses: actions/upload-artifact@v3 - if: failure() + - name: Upload FT repo screenshots + uses: actions/upload-artifact@v3 + if: failure() && (matrix.group < 10) with: name: ftr-cypress-screenshots path: ${{ env.FTR_PATH }}/cypress/screenshots retention-days: 1 - - - uses: actions/upload-artifact@v3 - if: always() + + - name: Upload FT repo videos + uses: actions/upload-artifact@v3 + if: always() && (matrix.group < 10) with: name: ftr-cypress-videos path: ${{ env.FTR_PATH }}/cypress/videos retention-days: 1 - - uses: actions/upload-artifact@v3 - if: always() + - name: Upload FT repo results + uses: actions/upload-artifact@v3 + if: always() && (matrix.group < 10) with: name: ftr-cypress-results path: ${{ env.FTR_PATH }}/cypress/results retention-days: 1 + - name: Upload Dashboards screenshots + if: failure() && (matrix.group > 9) + uses: actions/upload-artifact@v3 + with: + name: dashboards-cypress-screenshots + path: cypress/screenshots + retention-days: 1 + + - name: Upload Dashboards repo videos + uses: actions/upload-artifact@v3 + if: always() && (matrix.group > 9) + with: + name: dashboards-cypress-videos + path: cypress/videos + retention-days: 1 + + - name: Upload Dashboards repo results + uses: actions/upload-artifact@v3 + if: always() && (matrix.group > 9) + with: + name: dashboards-cypress-results + path: cypress/results + retention-days: 1 + add-comment: needs: [cypress-tests] if: ${{ always() && github.event_name == 'workflow_dispatch' && inputs.pr_number != '' }} diff --git a/.github/workflows/release_cypress_workflow.yml b/.github/workflows/release_cypress_workflow.yml new file mode 100644 index 000000000000..f58757b23f9f --- /dev/null +++ b/.github/workflows/release_cypress_workflow.yml @@ -0,0 +1,159 @@ +name: Orchestrator cypress workflow +run-name: release_cypress_workflow ${{ inputs.UNIQUE_ID != '' && inputs.UNIQUE_ID || '' }} # Unique id number appended to the workflow run-name to reference the run within the orchestrator. +# Trigger on dispatch event sent from FT repo orchestrator +on: + workflow_dispatch: + inputs: + test_repo: + description: 'Cypress test repo' + default: '' + required: false + type: string + test_branch: + description: 'Cypress test branch (default: source branch)' + required: false + type: string + specs: + description: 'Test group to run' + required: false + type: string + build_id: + description: 'Build Id' + required: false + type: string + OS_URL: + description: 'OpenSearch release artifact' + required: false + type: string + OSD_URL: + description: 'OpenSearch Dashboards release artifact' + required: false + type: string + UNIQUE_ID: + description: 'Unique Id for the workflow execution' + required: true + type: string + SECURITY_ENABLED: + required: false + type: string + +env: + TEST_REPO: ${{ inputs.test_repo != '' && inputs.test_repo || github.repository }} + TEST_BRANCH: "${{ inputs.test_branch != '' && inputs.test_branch || github.base_ref }}" + OSD_PATH: 'osd' + CYPRESS_BROWSER: 'electron' + JOB_ID: ${{ inputs.UNIQUE_ID}} + OPENSEARCH: ${{ inputs.OS_URL != '' && inputs.OS_URL || 'https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/$VERSION/latest/linux/x64/tar/dist/opensearch/opensearch-$VERSION-linux-x64.tar.gz' }} + DASHBOARDS: ${{ inputs.OSD_URL != '' && inputs.OSD_URL || 'https://ci.opensearch.org/ci/dbc/distribution-build-opensearch-dashboards/$VERSION/latest/linux/x64/tar/dist/opensearch-dashboards/opensearch-dashboards-$VERSION-linux-x64.tar.gz' }} + OPENSEARCH_DIR: 'cypress/opensearch' + DASHBOARDS_DIR: 'cypress/opensearch-dashboards' + SECURITY_ENABLED: ${{ inputs.SECURITY_ENABLED != '' && inputs.SECURITY_ENABLED || 'false' }} + +jobs: + Get-CI-Image-Tag: + uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main + with: + product: opensearch-dashboards + + cypress-tests: + needs: Get-CI-Image-Tag + runs-on: ubuntu-latest + strategy: + matrix: + spec_group: [10, 11] # Add more spec files as needed + container: + # using the same image which is used by opensearch-build team to build the OpenSearch Distribution + # this image tag is subject to change as more dependencies and updates will arrive over time + image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }} + # need to switch to root so that github actions can install runner binary on container without permission issues. + options: --user root + env: + # prevents extra Cypress installation progress messages + CI: 1 + # avoid warnings like "tput: No value for $TERM and no -T specified" + TERM: xterm + name: Run cypress tests (osd:ciGroup${{ matrix.spec_group }}) ${{ inputs.UNIQUE_ID}} + steps: + - name: Checkout code + uses: actions/checkout@v2 + with: + path: ./${{ env.OSD_PATH }} + repository: ${{ env.TEST_REPO }} + ref: '${{ env.TEST_BRANCH }}' + + - name: Setup Node + uses: actions/setup-node@v3 + with: + node-version-file: './${{ env.OSD_PATH }}/.nvmrc' + registry-url: 'https://registry.npmjs.org' + + - name: Setup Yarn + run: | + npm uninstall -g yarn + npm i -g yarn@1.22.10 + yarn config set network-timeout 1000000 -g + + - name: Setup spec files + if: ${{ inputs.specs == '' }} + shell: bash + run: | + cd ${{ env.OSD_PATH }} + IFS="," read -a SPEC_ARRAY <<< $(yarn --silent osd:ciGroup${{ matrix.spec_group }}) + FORMATTED_SPEC='' + for i in "${SPEC_ARRAY[@]}"; do + FORMATTED_SPEC+="cypress/integration/core_opensearch_dashboards/${i}," + done + SPEC=${FORMATTED_SPEC} # Set SPEC to FORMATTED_SPEC + echo "SPEC=${FORMATTED_SPEC}" >> $GITHUB_ENV + echo "SPEC found: $SPEC" + + - name: Get Cypress version + id: cypress_version + run: | + echo "name=cypress_version::$(cat ./${{ env.OSD_PATH }}/package.json | jq '.devDependencies.cypress' | tr -d '"')" >> $GITHUB_OUTPUT + + - name: Get package version (Linux) + run: | + cd ${{ env.OSD_PATH }} + echo "VERSION=$(yarn --silent pkg-version)" >> $GITHUB_ENV + + - name: Run bootstrap + run: | + cd ${{ env.OSD_PATH }} + chown -R 1000:1000 `pwd` + su `id -un 1000` -c "source $NVM_DIR/nvm.sh && nvm use && node -v && yarn -v && + yarn osd bootstrap" + + - name: Download and extract OpenSearch artifacts + run: | + CWD=$(pwd) + mkdir -p $CWD/${{ env.OPENSEARCH_DIR }} + source ${{ env.OSD_PATH }}/scripts/common/utils.sh + open_artifact $CWD/${{ env.OPENSEARCH_DIR }} ${{ env.OPENSEARCH }} + + - name: Download and extract OpenSearch Dashboards artifacts + run: | + CWD=$(pwd) + mkdir -p $CWD/${{ env.DASHBOARDS_DIR }} + source ${{ env.OSD_PATH }}/scripts/common/utils.sh + open_artifact $CWD/${{ env.DASHBOARDS_DIR }} ${{ env.DASHBOARDS }} + + - name: Run Cypress tests + run: | + chown -R 1000:1000 `pwd` + su `id -un 1000` -c "source ${{ env.OSD_PATH }}/scripts/cypress_tests.sh && run_dashboards_cypress_tests" + + # Screenshots are only captured on failures + - uses: actions/upload-artifact@v3 + if: failure() + with: + name: release-osd-cypress-screenshots + path: ${{ env.OSD_PATH }}/cypress/screenshots + retention-days: 1 + + - uses: actions/upload-artifact@v3 + if: always() + with: + name: release-osd-cypress-videos + path: ${{ env.OSD_PATH }}/cypress/videos + retention-days: 1 diff --git a/CHANGELOG.md b/CHANGELOG.md index 5251288d730e..a530cb067d18 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,8 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) ### 🔩 Tests +- Add functional test cypress workflow improvements and enable the workflow for in-house Dashboards tests ([#6061](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/6061)) + ## [2.12.0 - 2024-02-20](https://github.com/opensearch-project/OpenSearch-Dashboards/releases/tag/2.12.0) ### 💥 Breaking Changes diff --git a/cypress/utils/commands.js b/cypress/utils/commands.js new file mode 100644 index 000000000000..56a1fd0cff0e --- /dev/null +++ b/cypress/utils/commands.js @@ -0,0 +1,15 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +// --- Typed commands -- + +Cypress.Commands.add('getElementByTestId', (testId, options = {}) => { + return cy.get(`[data-test-subj="${testId}"]`, options); +}); + +Cypress.Commands.add('getElementsByTestIds', (testIds, options = {}) => { + const selectors = [testIds].flat(Infinity).map((testId) => `[data-test-subj="${testId}"]`); + return cy.get(selectors.join(','), options); +}); diff --git a/package.json b/package.json index 4dfffd211256..2a198f4d8fc7 100644 --- a/package.json +++ b/package.json @@ -77,8 +77,9 @@ "spec_to_console": "scripts/use_node scripts/spec_to_console", "pkg-version": "scripts/use_node -e \"console.log(require('./package.json').version)\"", "cypress:run-without-security": "env TZ=America/Los_Angeles NO_COLOR=1 cypress run --headless --env SECURITY_ENABLED=false", - "cypress:run-with-security": "env TZ=America/Los_Angeles NO_COLOR=1 cypress run --headless --env SECURITY_ENABLED=true,openSearchUrl=https://localhost:9200,WAIT_FOR_LOADER_BUFFER_MS=500" - + "cypress:run-with-security": "env TZ=America/Los_Angeles NO_COLOR=1 cypress run --headless --env SECURITY_ENABLED=true,openSearchUrl=https://localhost:9200,WAIT_FOR_LOADER_BUFFER_MS=500", + "osd:ciGroup10": "echo \"dashboard_sanity_test_spec.js\"", + "osd:ciGroup11": "echo \"apps/vis_builder/*.js\"" }, "repository": { "type": "git", diff --git a/scripts/common/opensearch_service.sh b/scripts/common/opensearch_service.sh index 09f3b5d50c8c..120caee12d26 100755 --- a/scripts/common/opensearch_service.sh +++ b/scripts/common/opensearch_service.sh @@ -9,6 +9,7 @@ function setup_opensearch() { cd "$OPENSEARCH_DIR" echo "network.host: 0.0.0.0" >> config/opensearch.yml echo "discovery.type: single-node" >> config/opensearch.yml + echo "cluster.routing.allocation.disk.threshold_enabled: false" >> config/opensearch.yml [ $SECURITY_ENABLED == "false" ] && [ -d "plugins/opensearch-security" ] && echo "plugins.security.disabled: true" >> config/opensearch.yml # Required for IM [ -d "plugins/opensearch-index-management" ] && echo "path.repo: [/tmp]" >> config/opensearch.yml diff --git a/scripts/cypress_tests.sh b/scripts/cypress_tests.sh index e4c2cbfc9352..93e58664f438 100644 --- a/scripts/cypress_tests.sh +++ b/scripts/cypress_tests.sh @@ -67,10 +67,12 @@ function check_status() { # Starts OpenSearch Dashboards and run tests in the cypress folder function run_dashboards_cypress_tests() { + run_opensearch + cd $CWD echo "[ OpenSearch Dashboards setup before it starts... ]" setup_dashboards >> /dev/null 2>&1 & sleep 100 - cd "$DASHBOARDS_DIR" + cd $CWD/"$DASHBOARDS_DIR" if [ -x "./bin/opensearch-dashboards" ]; then spawn_process_and_save_PID "./bin/opensearch-dashboards &" else @@ -79,11 +81,11 @@ function run_dashboards_cypress_tests() { fi check_status $DASHBOARDS_URL $DASHBOARDS_MSG # Run cypress tests + cd "$CWD"/osd run_cypress } function run_cypress() { - cd "$CWD"/osd echo "SPEC found: $SPEC" if [ $SECURITY_ENABLED = "true" ]; then echo "run security enabled tests" diff --git a/yarn.lock b/yarn.lock index fd15df82d494..9672e06948df 100644 --- a/yarn.lock +++ b/yarn.lock @@ -17220,7 +17220,7 @@ tar@^6.0.2, tar@^6.1.11: mkdirp "^1.0.3" yallist "^4.0.0" -tcp-port-used@^1.0.2: +tcp-port-used@^1.0.1: version "1.0.2" resolved "https://registry.yarnpkg.com/tcp-port-used/-/tcp-port-used-1.0.2.tgz#9652b7436eb1f4cfae111c79b558a25769f6faea" integrity sha512-l7ar8lLUD3XS1V2lfoJlCBaeoaWo/2xfYt81hM7VlvR4RrMVFqfmzfhLVk40hAb368uitje5gPtBRL1m/DGvLA==