WIP Mi/pw exp

.github/actions/setup-test-env/action.yml

@@ -47,3 +47,7 @@ runs:
       uses: ./.github/actions/install-r
       with:
         version: "4.4.0"
+
+    - name: Compile E2E Tests
+      shell: bash
+      run: yarn --cwd test/automation compile && yarn --cwd test/smoke compile

.github/workflows/positron-full-test.yml

@@ -95,7 +95,12 @@ jobs:
 
   e2e-electron-tests:
     runs-on: ubuntu-latest-8x
-    timeout-minutes: 80
+    timeout-minutes: 50
+    strategy:
+      fail-fast: false
+      matrix:
+        shardIndex: [1, 2]
+        shardTotal: [2]
     env:
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       POSITRON_BUILD_NUMBER: 0 # CI skips building releases
@@ -126,45 +131,30 @@ jobs:
         env:
           POSITRON_PY_VER_SEL: 3.10.12
           POSITRON_R_VER_SEL: 4.4.0
-        id: electron-smoke-tests
-        run: DISPLAY=:10 yarn smoketest-all --tracing --skip-cleanup
+        id: electron-tests
+        run: DISPLAY=:10 npx playwright test --project e2e-electron --workers 1 --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }}
 
-      - name: Convert XUnit to JUnit
-        id: xunit-to-junit
-        if: success() || failure()
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y libxml2-utils
-          yarn xunit-to-junit smoke-tests-electron
-
-      - name: Publish Electron Test Report
-        uses: mikepenz/action-junit-report@v4
-        if: success() || failure()
+      - name: Upload blob report
+        if: ${{ !cancelled() }}
+        uses: actions/upload-artifact@v4
         with:
-          report_paths: "**/.build/logs/smoke-tests-electron/test-results/xunit-results.xml"
-          check_name: "Electron Test Results"
-          token: ${{ secrets.GITHUB_TOKEN }}
+          name: blob-report-electron-${{ matrix.shardIndex }}
+          path: blob-report
+          retention-days: 14
 
       - name: Set TestRail Run Title
         id: set-testrail-run-title
-        if: steps.xunit-to-junit == 'success'
-        run: echo "TESTRAIL_TITLE=$(date +'%Y-%m-%d') Nightly Smoke Tests - $GITHUB_REF_NAME" >> $GITHUB_ENV
+        if: ${{ !cancelled() }}
+        run: echo "TESTRAIL_TITLE=$(date +'%Y-%m-%d') Nightly E2E Electron Tests - $GITHUB_REF_NAME" >> $GITHUB_ENV
 
       - name: Upload Test Results to TestRail (Electron ONLY)
         id: testrail-upload
-        if: steps.xunit-to-junit == 'success'
-        run: trcli --host "https://posit.testrail.io/" --project Positron --username testrailautomation@posit.co --key ${{ secrets.TESTRAIL_API_KEY}} parse_junit --file ".build/logs/smoke-tests-electron/test-results/xunit-results.xml" --case-matcher name --title "$TESTRAIL_TITLE" --close-run
-
-      - name: Upload Artifacts - Electron
-        if: always()
-        uses: actions/upload-artifact@v4
-        with:
-          name: run-artifacts-electron
-          path: .build/logs/smoke-tests-electron/
+        if: ${{ !cancelled() }}
+        run: pip install trcli && trcli --host "https://posit.testrail.io/" --project Positron --username testrailautomation@posit.co --key ${{ secrets.TESTRAIL_API_KEY}} parse_junit --file "./test-results/junit.xml" --case-matcher name --title "$TESTRAIL_TITLE" --close-run
 
   e2e-browser-tests:
     runs-on: ubuntu-latest-4x
-    timeout-minutes: 40
+    timeout-minutes: 50
     env:
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       POSITRON_BUILD_NUMBER: 0 # CI skips building releases
@@ -197,42 +187,28 @@ jobs:
           aws-region: ${{ secrets.QA_AWS_REGION }}
           github-token: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Run Web Smoke Tests
+      - name: Run Tests (Browser)
         env:
           POSITRON_PY_VER_SEL: 3.10.12
           POSITRON_R_VER_SEL: 4.4.0
-        id: electron-web-smoke-tests
-        run: DISPLAY=:10 yarn smoketest-web --tracing
+        id: browser-tests
+        run: DISPLAY=:10 npx playwright test --project e2e-browser --workers 1
 
-      - name: Convert XUnit to JUnit
-        id: xunit-to-junit
-        if: success() || failure()
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y libxml2-utils
-          yarn xunit-to-junit smoke-tests-browser
-
-      - name: Publish Web Test Report
-        uses: mikepenz/action-junit-report@v4
-        if: success() || failure()
-        with:
-          report_paths: "**/.build/logs/smoke-tests-browser/test-results/xunit-results.xml"
-          check_name: "Web Test Results"
-          token: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Upload Artifacts - Browser
-        if: always()
+      - name: Upload blob report
+        if: ${{ !cancelled() }}
         uses: actions/upload-artifact@v4
         with:
-          name: run-artifacts-browser
-          path: .build/logs/smoke-tests-browser/
+          name: blob-report-browser
+          path: blob-report
+          retention-days: 14
 
       - name: Clean up license files
         if: always()
         run: cd .. && rm -rf positron-license
 
-  slack-notification:
-    name: "slack-notification"
+  e2e-report:
+    container:
+      image: mcr.microsoft.com/playwright:v1.48.0-jammy
     runs-on: ubuntu-latest
     needs:
       [
@@ -241,9 +217,48 @@ jobs:
         "e2e-electron-tests",
         "e2e-browser-tests",
       ]
-    if: failure()
+    if: ${{ !cancelled() }}
+    env:
+      AWS_S3_BUCKET: positron-test-reports
+      REPORT_DIR: playwright-report-${{ github.run_id }}
+
     steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version-file: .nvmrc
+
+      - name: Download blob reports
+        uses: actions/download-artifact@v4
+        with:
+          path: all-blob-reports
+          pattern: blob-report-*
+          merge-multiple: true
+
+      - name: Merge blobs into HTML Report
+        run: npx playwright merge-reports --reporter html ./all-blob-reports
+
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: ${{ secrets.AWS_TEST_REPORTS_ROLE }}
+          aws-region: "us-east-1"
+
+      - name: Upload Playwright Report to S3 Bucket
+        if: ${{ !cancelled() }}
+        run: aws s3 cp playwright-report/. s3://${{ env.AWS_S3_BUCKET }}/${{ env.REPORT_DIR }} --recursive
+
+      - name: Send HTML report URL to GitHub Summary
+        if: ${{ !cancelled() }}
+        run: |
+          REPORT_URL="https://${{ env.AWS_S3_BUCKET }}.s3.us-east-1.amazonaws.com/${{ env.REPORT_DIR }}/index.html"
+          echo $REPORT_URL > url.txt
+          echo "Report URL: $REPORT_URL"
+          echo "📄 [Playwright Report]($REPORT_URL) <br>" > $GITHUB_STEP_SUMMARY
+
       - name: "Send Slack notification"
+        if: ${{ failure() }}
         uses: testlabauto/action-test-results-to-slack@v0.0.6
         with:
           github_token: ${{ secrets.POSITRON_GITHUB_PAT }}

.github/workflows/positron-merge-to-main.yml

@@ -6,31 +6,86 @@ on:
       - main
   workflow_call:
     inputs:
-      smoketest_target:
+      e2e_grep:
         required: false
-        description: "Smoketest suite to run, e.g. smoketest-all or smoketest-pr"
-        default: "smoketest-all"
+        description: "Grep filter to apply to the e2e tests: @pr, @win, etc."
+        default: ""
         type: string
   workflow_dispatch:
     inputs:
-      smoketest_target:
+      e2e_grep:
         required: false
-        description: "Smoketest suite to run, e.g. smoketest-all or smoketest-pr"
-        default: "smoketest-all"
+        description: "Grep filter to apply to the e2e tests: @pr, @win, etc."
+        default: ""
         type: string
 
 env:
-  SMOKETEST_TARGET: ${{ inputs.smoketest_target || 'smoketest-all' }}
+  E2E_GREP: ${{ inputs.e2e_grep || '' }}
+  AWS_S3_BUCKET: positron-test-reports
+  REPORT_DIR: playwright-report-${{ github.run_id }}
 
 permissions:
   id-token: write
   contents: read
 
 jobs:
-  linux:
-    name: Tests on Linux
+  e2e-electron:
+    timeout-minutes: 30
     runs-on: ubuntu-latest-8x
-    timeout-minutes: 45
+    env:
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      POSITRON_BUILD_NUMBER: 0 # CI skips building releases
+      _R_CHECK_FUTURE_FILE_TIMESTAMPS_: false # this check can be flaky in the R pkg tests
+      _R_CHECK_CRAN_INCOMING_: false
+      _R_CHECK_SYSTEM_CLOCK_: false
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version-file: .nvmrc
+
+      - name: Cache node_modules, build, extensions, and remote
+        uses: ./.github/actions/cache-multi-paths
+
+      - name: Setup Build and Compile
+        uses: ./.github/actions/setup-build-env
+
+      - name: Setup E2E Test Environment
+        uses: ./.github/actions/setup-test-env
+        with:
+          aws-role-to-assume: ${{ secrets.QA_AWS_RO_ROLE }}
+          aws-region: ${{ secrets.QA_AWS_REGION }}
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Send HTML report URL to GitHub Summary
+        if: ${{ !cancelled() }}
+        run: |
+          REPORT_URL="https://${{ env.AWS_S3_BUCKET }}.s3.us-east-1.amazonaws.com/${{ env.REPORT_DIR }}/index.html"
+          echo $REPORT_URL > url.txt
+          echo "Report URL: $REPORT_URL"
+          echo "📄 [Playwright Report]($REPORT_URL) <br>" > $GITHUB_STEP_SUMMARY
+
+      - name: Run Playwright Tests (Electron)
+        env:
+          POSITRON_PY_VER_SEL: 3.10.12
+          POSITRON_R_VER_SEL: 4.4.0
+        id: e2e-playwright-tests
+        run: DISPLAY=:10 npx playwright test --project e2e-electron --workers 2 --grep="${{ env.E2E_GREP }}"
+
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: ${{ secrets.AWS_TEST_REPORTS_ROLE }}
+          aws-region: "us-east-1"
+
+      - name: Upload Playwright Report to S3 Bucket
+        if: ${{ !cancelled() }}
+        run: aws s3 cp playwright-report/. s3://${{ env.AWS_S3_BUCKET }}/$REPORT_DIR --recursive
+
+  unit-integration:
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
     env:
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       POSITRON_BUILD_NUMBER: 0 # CI skips building releases
@@ -56,12 +111,11 @@ jobs:
           github-token: ${{ secrets.POSITRON_GITHUB_PAT }}
           license-key: ${{ secrets.POSITRON_DEV_LICENSE }}
 
-      - name: Setup E2E Test Environment
-        uses: ./.github/actions/setup-test-env
+      # one unit test needs this: Can list tables and fields from R connections
+      - name: Setup R
+        uses: ./.github/actions/install-r
         with:
-          aws-role-to-assume: ${{ secrets.QA_AWS_RO_ROLE }}
-          aws-region: ${{ secrets.QA_AWS_REGION }}
-          github-token: ${{ secrets.GITHUB_TOKEN }}
+          version: "4.4.0"
 
       - name: Compile Integration Tests
         run: yarn --cwd test/integration/browser compile
@@ -74,47 +128,11 @@ jobs:
         id: electron-integration-tests
         run: DISPLAY=:10 ./scripts/test-integration-pr.sh
 
-      - name: Run Smoke Tests (Electron)
-        if: success() || failure()
-        env:
-          POSITRON_PY_VER_SEL: 3.10.12
-          POSITRON_R_VER_SEL: 4.4.0
-        id: electron-smoke-tests
-        run: DISPLAY=:10 yarn ${{ env.SMOKETEST_TARGET }} --tracing --parallel --jobs 2 --skip-cleanup
-
-      - name: Convert XUnit to JUnit
-        id: xunit-to-junit
-        if: success() || failure()
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y libxml2-utils
-          /usr/bin/xmllint --version
-          yarn xunit-to-junit smoke-tests-electron
-
-      - name: Set TestRail Run Title
-        id: set-testrail-run-title
-        if: always()
-        run: echo "TESTRAIL_TITLE=$(date +'%Y-%m-%d') Smoke Tests on branch $GITHUB_REF_NAME" >> $GITHUB_ENV
-
-      - name: Upload Test Results to TestRail
-        id: testrail-upload
-        if: always()
-        run: trcli --host "https://posit.testrail.io/" --project Positron --username testrailautomation@posit.co --key ${{ secrets.TESTRAIL_API_KEY}} parse_junit --file ".build/logs/smoke-tests-electron/test-results/results.xml" --case-matcher name --title "$TESTRAIL_TITLE" --close-run
-
-      - name: Upload run artifacts
-        if: always()
-        uses: actions/upload-artifact@v4
-        with:
-          name: run-artifacts
-          path: .build/logs/smoke-tests-electron/
-    outputs:
-      target: ${{ env.SMOKETEST_TARGET }}
-
   slack-notification:
     name: "Send Slack notification"
     runs-on: ubuntu-latest
-    needs: linux
-    if: ${{ failure() && needs.linux.outputs.target  == 'smoketest-all' }}
+    needs: [unit-integration, e2e-electron]
+    if: ${{ failure() && inputs.e2e_grep == '' }}
     steps:
       - name: "Send Slack notification"
         uses: testlabauto/action-test-results-to-slack@v0.0.6

.github/workflows/positron-pull-requests.yml

@@ -6,8 +6,8 @@ on:
       - main
 
 jobs:
-  positron-ci:
+  tests:
     uses: ./.github/workflows/positron-merge-to-main.yml
     secrets: inherit
     with:
-      smoketest_target: smoketest-pr
+      e2e_grep: "@pr"