Add 2 new GitHub Actions workflows to include COT E2E and API automat…

…ed tests in CI. (woocommerce#34615) * Add 2 new GitHub Actions workflows to include COT E2E and API automated tests in CI. * Fix typo in GitHub Actions workflow name. * Update new PR COT test workflow to trigger with any PR. * Update new cot-pr-build-and-e2e-tests.yml GitHub Actions workflow to use the Monorepo Setup Action. * Updated 'cot-build-and-e2e-tests-daily.yml' to use the new WooCommerce Monorepo action and 'cot-pr-build-and-e2e-tests.yml' to trigger on PRs labeled with the custom order tables tag.
wpexplorer · Sep 19, 2022 · aaff1c2 · aaff1c2
1 parent 8478993
commit aaff1c2
Show file tree

Hide file tree

Showing 2 changed files with 382 additions and 0 deletions.
diff --git a/.github/workflows/cot-build-and-e2e-tests-daily.yml b/.github/workflows/cot-build-and-e2e-tests-daily.yml
@@ -0,0 +1,190 @@
+name: Run daily tests in an environment with COT enabled
+on:
+  schedule:
+    - cron: "30 2 * * *"
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+    cot-e2e-tests-run:
+        name: Runs E2E tests with COT enabled.
+        runs-on: ubuntu-20.04
+        steps:
+            - uses: actions/checkout@v3
+
+            - name: Setup WooCommerce Monorepo
+              uses: ./.github/actions/setup-woocommerce-monorepo
+
+            - name: Load docker images and start containers with COT enabled.
+              working-directory: plugins/woocommerce
+              run: pnpm env:test:cot --filter=woocommerce
+
+            - name: Download and install Chromium browser.
+              working-directory: plugins/woocommerce
+              run: pnpx playwright install chromium
+
+            - name: Run Playwright E2E tests.
+              timeout-minutes: 60
+              id: run_playwright_e2e_tests
+              env:
+                USE_WP_ENV: 1
+              working-directory: plugins/woocommerce
+              run: pnpx playwright test --config=tests/e2e-pw/playwright.config.js
+
+            - name: Generate Playwright E2E Test report.
+              id: generate_e2e_report
+              if: |
+                  always() &&
+                  (
+                    steps.run_playwright_e2e_tests.conclusion != 'cancelled' ||
+                    steps.run_playwright_e2e_tests.conclusion != 'skipped' 
+                  )
+              working-directory: plugins/woocommerce
+              run: pnpx allure generate --clean e2e/allure-results --output e2e/allure-report
+
+            - name: Archive Playwright E2E test report
+              if: |
+                  always() &&
+                  steps.generate_e2e_report.conclusion == 'success'
+              uses: actions/upload-artifact@v3
+              with:
+                  name: e2e-test-report---pr-${{ github.event.number }}
+                  path: |
+                      plugins/woocommerce/e2e/allure-results
+                      plugins/woocommerce/e2e/allure-report
+                  if-no-files-found: ignore
+                  retention-days: 5
+
+    cot-api-tests-run:
+        name: Runs API tests with COT enabled.
+        runs-on: ubuntu-20.04
+        env:
+            API_TEST_REPORT_DIR: ${{ github.workspace }}/api-test-report
+        steps:
+            - uses: actions/checkout@v3
+
+            - name: Setup WooCommerce Monorepo
+              uses: ./.github/actions/setup-woocommerce-monorepo
+
+            - name: Load docker images and start containers with COT enabled.
+              working-directory: plugins/woocommerce
+              run: pnpm env:test:cot --filter=woocommerce
+
+            - name: Run tests command.
+              working-directory: plugins/woocommerce
+              env:
+                  BASE_URL: http://localhost:8086
+                  USER_KEY: admin
+                  USER_SECRET: password
+              run: pnpm exec wc-api-tests test api
+
+            - name: Archive API test report
+              if: always()
+              uses: actions/upload-artifact@v3
+              with:
+                  name: api-test-report---pr-${{ github.event.number }}
+                  path: |
+                      ${{ env.API_TEST_REPORT_DIR }}/allure-results
+                      ${{ env.API_TEST_REPORT_DIR }}/allure-report
+                  retention-days: 5
+
+    test-summary:
+        name: Post test results
+        if: |
+            always() && 
+            ! github.event.pull_request.head.repo.fork &&
+            (
+              contains( needs.*.result, 'success' ) ||
+              contains( needs.*.result, 'failure' )
+            )
+        runs-on: ubuntu-20.04
+        needs: [cot-api-tests-run, cot-e2e-tests-run]
+        steps:
+            - name: Create dirs
+              run: |
+                  mkdir -p repo
+                  mkdir -p artifacts/api 
+                  mkdir -p artifacts/e2e
+                  mkdir -p output
+
+            - name: Checkout code
+              uses: actions/checkout@v3
+              with:
+                  path: repo
+
+            - name: Download API test report artifact
+              uses: actions/download-artifact@v3
+              with:
+                  name: api-test-report---pr-${{ github.event.number }}
+                  path: artifacts/api
+
+            - name: Download Playwright E2E test report artifact
+              uses: actions/download-artifact@v3
+              with:
+                  name: e2e-test-report---pr-${{ github.event.number }}
+                  path: artifacts/e2e
+
+            - name: Prepare test summary
+              id: prepare-test-summary
+              uses: actions/github-script@v6
+              env:
+                  API_SUMMARY_PATH: ${{ github.workspace }}/artifacts/api/allure-report/widgets/summary.json
+                  E2E_PW_SUMMARY_PATH: ${{ github.workspace }}/artifacts/e2e/allure-report/widgets/summary.json
+                  E2E_PPTR_SUMMARY_PATH: ${{ github.workspace }}/artifacts/e2e/test-results.json
+                  PR_NUMBER: ${{ github.event.number }}
+                  SHA: ${{ github.event.pull_request.head.sha }}
+              with:
+                  result-encoding: string
+                  script: |
+                      const script = require( './repo/.github/workflows/scripts/prepare-test-summary.js' )
+                      return await script( { core } )
+
+            - name: Find PR comment by github-actions[bot]
+              uses: peter-evans/find-comment@v2
+              id: find-comment
+              with:
+                  issue-number: ${{ github.event.pull_request.number }}
+                  comment-author: 'github-actions[bot]'
+                  body-includes: Test Results Summary
+
+            - name: Create or update PR comment
+              uses: peter-evans/create-or-update-comment@v2
+              with:
+                  comment-id: ${{ steps.find-comment.outputs.comment-id }}
+                  issue-number: ${{ github.event.pull_request.number }}
+                  body: ${{ steps.prepare-test-summary.outputs.result }}
+                  edit-mode: replace
+
+    publish-test-reports:
+        name: Publish test reports
+        if: |
+            always() && 
+            ! github.event.pull_request.head.repo.fork &&
+            (
+              contains( needs.*.result, 'success' ) ||
+              contains( needs.*.result, 'failure' )
+            )
+        runs-on: ubuntu-20.04
+        needs: [cot-api-tests-run, cot-e2e-tests-run]
+        env:
+            GITHUB_TOKEN: ${{ secrets.REPORTS_TOKEN }}
+            PR_NUMBER: ${{ github.event.number }}
+            RUN_ID: ${{ github.run_id }}
+            COMMIT_SHA: ${{ github.event.pull_request.head.sha }}
+        steps:
+            - name: Publish test reports
+              env:
+                  API_ARTIFACT: api-test-report---pr-${{ github.event.number }}
+                  E2E_ARTIFACT: e2e-test-report---pr-${{ github.event.number }}
+              run: |
+                  gh workflow run publish-test-reports-pr.yml \
+                    -f run_id=$RUN_ID \
+                    -f api_artifact=$API_ARTIFACT \
+                    -f e2e_artifact=$E2E_ARTIFACT \
+                    -f pr_number=$PR_NUMBER \
+                    -f commit_sha=$COMMIT_SHA \
+                    -f s3_root=public \
+                    --repo woocommerce/woocommerce-test-reports
diff --git a/.github/workflows/cot-pr-build-and-e2e-tests.yml b/.github/workflows/cot-pr-build-and-e2e-tests.yml
@@ -0,0 +1,192 @@
+name: Run tests against PR in an environment with COT enabled
+on:
+    pull_request:
+        types: [labeled]
+    workflow_dispatch:
+
+concurrency:
+    group: ${{ github.workflow }}-${{ github.ref }}
+    cancel-in-progress: true
+
+jobs:
+    cot-e2e-tests-run:
+        name: Runs E2E tests with COT enabled.
+        if: "${{ github.event_name == 'workflow_dispatch' || github.event.label.name == 'focus: custom order tables' }}"
+        runs-on: ubuntu-20.04
+        steps:
+            - uses: actions/checkout@v3
+
+            - name: Setup WooCommerce Monorepo
+              uses: ./.github/actions/setup-woocommerce-monorepo
+
+            - name: Load docker images and start containers with COT enabled.
+              working-directory: plugins/woocommerce
+              run: pnpm env:test:cot --filter=woocommerce
+
+            - name: Download and install Chromium browser.
+              working-directory: plugins/woocommerce
+              run: pnpx playwright install chromium
+
+            - name: Run Playwright E2E tests.
+              timeout-minutes: 60
+              id: run_playwright_e2e_tests
+              env:
+                USE_WP_ENV: 1
+              working-directory: plugins/woocommerce
+              run: pnpx playwright test --config=tests/e2e-pw/playwright.config.js
+
+            - name: Generate Playwright E2E Test report.
+              id: generate_e2e_report
+              if: |
+                  always() &&
+                  (
+                    steps.run_playwright_e2e_tests.conclusion != 'cancelled' ||
+                    steps.run_playwright_e2e_tests.conclusion != 'skipped' 
+                  )
+              working-directory: plugins/woocommerce
+              run: pnpx allure generate --clean e2e/allure-results --output e2e/allure-report
+
+            - name: Archive Playwright E2E test report
+              if: |
+                  always() &&
+                  steps.generate_e2e_report.conclusion == 'success'
+              uses: actions/upload-artifact@v3
+              with:
+                  name: e2e-test-report---pr-${{ github.event.number }}
+                  path: |
+                      plugins/woocommerce/e2e/allure-results
+                      plugins/woocommerce/e2e/allure-report
+                  if-no-files-found: ignore
+                  retention-days: 5
+
+    cot-api-tests-run:
+        name: Runs API tests with COT enabled.
+        if: "${{ github.event_name == 'workflow_dispatch' || github.event.label.name == 'focus: custom order tables' }}"
+        runs-on: ubuntu-20.04
+        env:
+            API_TEST_REPORT_DIR: ${{ github.workspace }}/api-test-report
+        steps:
+            - uses: actions/checkout@v3
+
+            - name: Setup WooCommerce Monorepo
+              uses: ./.github/actions/setup-woocommerce-monorepo
+
+            - name: Load docker images and start containers with COT enabled.
+              working-directory: plugins/woocommerce
+              run: pnpm env:test:cot --filter=woocommerce
+
+            - name: Run tests command.
+              working-directory: plugins/woocommerce
+              env:
+                  BASE_URL: http://localhost:8086
+                  USER_KEY: admin
+                  USER_SECRET: password
+              run: pnpm exec wc-api-tests test api
+
+            - name: Archive API test report
+              if: always()
+              uses: actions/upload-artifact@v3
+              with:
+                  name: api-test-report---pr-${{ github.event.number }}
+                  path: |
+                      ${{ env.API_TEST_REPORT_DIR }}/allure-results
+                      ${{ env.API_TEST_REPORT_DIR }}/allure-report
+                  retention-days: 5
+
+    test-summary:
+        name: Post test results
+        if: |
+            always() && 
+            ! github.event.pull_request.head.repo.fork &&
+            (
+              contains( needs.*.result, 'success' ) ||
+              contains( needs.*.result, 'failure' )
+            )
+        runs-on: ubuntu-20.04
+        needs: [cot-api-tests-run, cot-e2e-tests-run]
+        steps:
+            - name: Create dirs
+              run: |
+                  mkdir -p repo
+                  mkdir -p artifacts/api 
+                  mkdir -p artifacts/e2e
+                  mkdir -p output
+
+            - name: Checkout code
+              uses: actions/checkout@v3
+              with:
+                  path: repo
+
+            - name: Download API test report artifact
+              uses: actions/download-artifact@v3
+              with:
+                  name: api-test-report---pr-${{ github.event.number }}
+                  path: artifacts/api
+
+            - name: Download Playwright E2E test report artifact
+              uses: actions/download-artifact@v3
+              with:
+                  name: e2e-test-report---pr-${{ github.event.number }}
+                  path: artifacts/e2e
+
+            - name: Prepare test summary
+              id: prepare-test-summary
+              uses: actions/github-script@v6
+              env:
+                  API_SUMMARY_PATH: ${{ github.workspace }}/artifacts/api/allure-report/widgets/summary.json
+                  E2E_PW_SUMMARY_PATH: ${{ github.workspace }}/artifacts/e2e/allure-report/widgets/summary.json
+                  E2E_PPTR_SUMMARY_PATH: ${{ github.workspace }}/artifacts/e2e/test-results.json
+                  PR_NUMBER: ${{ github.event.number }}
+                  SHA: ${{ github.event.pull_request.head.sha }}
+              with:
+                  result-encoding: string
+                  script: |
+                      const script = require( './repo/.github/workflows/scripts/prepare-test-summary.js' )
+                      return await script( { core } )
+
+            - name: Find PR comment by github-actions[bot]
+              uses: peter-evans/find-comment@v2
+              id: find-comment
+              with:
+                  issue-number: ${{ github.event.pull_request.number }}
+                  comment-author: 'github-actions[bot]'
+                  body-includes: Test Results Summary
+
+            - name: Create or update PR comment
+              uses: peter-evans/create-or-update-comment@v2
+              with:
+                  comment-id: ${{ steps.find-comment.outputs.comment-id }}
+                  issue-number: ${{ github.event.pull_request.number }}
+                  body: ${{ steps.prepare-test-summary.outputs.result }}
+                  edit-mode: replace
+
+    publish-test-reports:
+        name: Publish test reports
+        if: |
+            always() && 
+            ! github.event.pull_request.head.repo.fork &&
+            (
+              contains( needs.*.result, 'success' ) ||
+              contains( needs.*.result, 'failure' )
+            )
+        runs-on: ubuntu-20.04
+        needs: [cot-api-tests-run, cot-e2e-tests-run]
+        env:
+            GITHUB_TOKEN: ${{ secrets.REPORTS_TOKEN }}
+            PR_NUMBER: ${{ github.event.number }}
+            RUN_ID: ${{ github.run_id }}
+            COMMIT_SHA: ${{ github.event.pull_request.head.sha }}
+        steps:
+            - name: Publish test reports
+              env:
+                  API_ARTIFACT: api-test-report---pr-${{ github.event.number }}
+                  E2E_ARTIFACT: e2e-test-report---pr-${{ github.event.number }}
+              run: |
+                  gh workflow run publish-test-reports-pr.yml \
+                    -f run_id=$RUN_ID \
+                    -f api_artifact=$API_ARTIFACT \
+                    -f e2e_artifact=$E2E_ARTIFACT \
+                    -f pr_number=$PR_NUMBER \
+                    -f commit_sha=$COMMIT_SHA \
+                    -f s3_root=public \
+                    --repo woocommerce/woocommerce-test-reports