ok-to-test-command #20111
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Appsmith External Integration Test Workflow | |
| on: | |
| # This workflow is only triggered by the ok to test command dispatch | |
| repository_dispatch: | |
| types: [ok-to-test-command] | |
| jobs: | |
| notify: | |
| runs-on: ubuntu-latest | |
| steps: | |
| # This step creates a comment on the PR with a link to this workflow run. | |
| - name: Add a comment on the PR with link to workflow run | |
| uses: peter-evans/create-or-update-comment@v2 | |
| with: | |
| issue-number: ${{ github.event.client_payload.pull_request.number }} | |
| body: | | |
| Tests running at: <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}>. | |
| Workflow: `${{ github.workflow }}`. | |
| Commit: `${{ github.event.client_payload.slash_command.args.named.sha }}`. | |
| PR: ${{ github.event.client_payload.pull_request.number }}. | |
| server-build: | |
| name: server-build | |
| uses: ./.github/workflows/server-build.yml | |
| secrets: inherit | |
| with: | |
| pr: ${{ github.event.client_payload.pull_request.number }} | |
| skip-tests: true | |
| client-build: | |
| name: client-build | |
| uses: ./.github/workflows/client-build.yml | |
| secrets: inherit | |
| with: | |
| pr: ${{ github.event.client_payload.pull_request.number }} | |
| check-test-files: "true" | |
| rts-build: | |
| name: rts-build | |
| uses: ./.github/workflows/rts-build.yml | |
| secrets: inherit | |
| with: | |
| pr: ${{ github.event.client_payload.pull_request.number }} | |
| test-appsmithctl: | |
| name: appsmithctl | |
| uses: ./.github/workflows/appsmithctl.yml | |
| secrets: inherit | |
| with: | |
| pr: ${{ github.event.client_payload.pull_request.number }} | |
| build-docker-image: | |
| needs: [client-build, server-build, rts-build] | |
| # Only run if the build step is successful | |
| if: success() | |
| name: build-docker-image | |
| uses: ./.github/workflows/build-docker-image.yml | |
| secrets: inherit | |
| with: | |
| pr: ${{ github.event.client_payload.pull_request.number }} | |
| ci-test: | |
| needs: [build-docker-image] | |
| # Only run if the build step is successful | |
| if: success() | |
| name: ci-test | |
| uses: ./.github/workflows/ci-test-custom-script.yml | |
| secrets: inherit | |
| with: | |
| pr: ${{ github.event.client_payload.pull_request.number }} | |
| ci-test-result: | |
| needs: [ci-test] | |
| # Only run if the ci-test with matrices step is successful | |
| if: always() | |
| runs-on: ubuntu-latest | |
| defaults: | |
| run: | |
| shell: bash | |
| steps: | |
| - name: Dump the client payload context | |
| env: | |
| PAYLOAD_CONTEXT: ${{ toJson(github.event.client_payload) }} | |
| run: echo "$PAYLOAD_CONTEXT" | |
| - name: Setup node | |
| if: needs.ci-test.result != 'success' | |
| uses: actions/setup-node@v3 | |
| with: | |
| node-version: 18 | |
| - name: install pg | |
| if: needs.ci-test.result != 'success' | |
| run: npm install pg | |
| - name: Fetch the failed specs | |
| if: needs.ci-test.result != 'success' | |
| id: failed_specs | |
| env: | |
| DB_HOST: ${{ secrets.CYPRESS_DB_HOST }} | |
| DB_NAME: ${{ secrets.CYPRESS_DB_NAME }} | |
| DB_USER: ${{ secrets.CYPRESS_DB_USER }} | |
| DB_PWD: ${{ secrets.CYPRESS_DB_PWD }} | |
| RUN_ID: ${{ github.run_id }} | |
| ATTEMPT_NUMBER: ${{ github.run_attempt }} | |
| uses: actions/github-script@v6 | |
| with: | |
| script: | | |
| const { Pool } = require("pg"); | |
| const { DB_HOST, DB_NAME, DB_USER, DB_PWD, RUN_ID, ATTEMPT_NUMBER } = process.env | |
| const client = await new Pool({ | |
| user: DB_USER, | |
| host: DB_HOST, | |
| database: DB_NAME, | |
| password: DB_PWD, | |
| port: 5432, | |
| connectionTimeoutMillis: 60000, | |
| }).connect(); | |
| const result = await client.query( | |
| `SELECT DISTINCT name FROM public."specs" | |
| WHERE "matrixId" IN | |
| (SELECT id FROM public."matrix" | |
| WHERE "attemptId" = ( | |
| SELECT id FROM public."attempt" WHERE "workflowId" = $1 and "attempt" = $2 | |
| ) | |
| ) AND status = 'fail'`, | |
| [RUN_ID, ATTEMPT_NUMBER], | |
| ); | |
| client.release(); | |
| return result.rows.map((spec) => spec.name); | |
| # In case for any ci job failure, create combined failed spec | |
| - name: combine all specs for CI | |
| id: combine_ci | |
| if: needs.ci-test.result != 'success' | |
| run: | | |
| failed_specs=$(echo ${{steps.failed_specs.outputs.result}} | sed 's/\[\|\]//g' | tr -d ' ' | tr ',' '\n') | |
| while read -r line; do | |
| echo "$line" >> ~/combined_failed_spec_ci | |
| done <<< "$failed_specs" | |
| if [[ -z $(grep '[^[:space:]]' ~/combined_failed_spec_ci) ]] ; then | |
| echo "specs_failed=0" >> $GITHUB_OUTPUT | |
| else | |
| echo "specs_failed=1" >> $GITHUB_OUTPUT | |
| fi | |
| # Upload combined failed CI spec list to a file | |
| # This is done for debugging. | |
| - name: upload combined failed spec | |
| if: needs.ci-test.result != 'success' | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: combined_failed_spec_ci | |
| path: ~/combined_failed_spec_ci | |
| - name: Get Latest flaky Tests | |
| shell: bash | |
| run: | | |
| curl --request POST --url https://yatin-s-workspace-jk8ru5.us-east-1.xata.sh/db/CypressKnownFailures:main/tables/CypressKnownFailuires/query --header 'Authorization: Bearer ${{ secrets.XATA_TOKEN }}' --header 'Content-Type: application/json'|jq -r |grep Spec|cut -d ':' -f 2 2> /dev/null|sed 's/"//g'|sed 's/,//g' > ~/knownfailures | |
| # Verify CI test failures against known failures | |
| - name: Verify CI test failures against known failures | |
| if: needs.ci-test.result != 'success' | |
| shell: bash | |
| run: | | |
| new_failed_spec_env="<ol>$(comm -1 -3 <(sort ~/knownfailures) <(sort -u ~/combined_failed_spec_ci) | sed 's/|cypress|cypress/\n/g' | sed 's/^/<li>/')</ol>" | |
| echo "$new_failed_spec_env" | |
| echo "new_failed_spec_env<<EOF" >> $GITHUB_ENV | |
| echo "$new_failed_spec_env" >> $GITHUB_ENV | |
| echo "EOF" >> $GITHUB_ENV | |
| - name: Add a comment on the PR with new CI failures | |
| if: needs.ci-test.result != 'success' && steps.combine_ci.outputs.specs_failed == '1' | |
| uses: peter-evans/create-or-update-comment@v1 | |
| with: | |
| issue-number: ${{ github.event.client_payload.pull_request.number }} | |
| body: | | |
| Workflow run: <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}>. | |
| Commit: `${{ github.event.client_payload.slash_command.args.named.sha }}`. | |
| Cypress dashboard: <a href="https://internal.appsmith.com/app/cypressdashboard/rundetails-64ec3df0c632e24c00764938?branch=master&workflowId=${{ github.run_id }}&attempt=${{ github.run_attempt }}&selectiontype=test&testsstatus=failed&specsstatus=fail" target="_blank"> Click here!</a> | |
| The following are new failures, please fix them before merging the PR: ${{env.new_failed_spec_env}} | |
| To know the list of identified flaky tests - <a href="https://internal.appsmith.com/app/cypressdashboard/identified-flaky-tests-6541e22b0c243558fe2288ca?branch=master" target="_blank">Refer here</a> | |
| - name: Add a comment on the PR when ci-test is failed but no specs found | |
| if: needs.ci-test.result != 'success' && steps.combine_ci.outputs.specs_failed == '0' | |
| uses: peter-evans/create-or-update-comment@v1 | |
| with: | |
| issue-number: ${{ github.event.client_payload.pull_request.number }} | |
| body: | | |
| Workflow run: <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}>. | |
| Commit: `${{ github.event.client_payload.slash_command.args.named.sha }}`. | |
| Cypress dashboard url: <a href="https://internal.appsmith.com/app/cypressdashboard/rundetails-64ec3df0c632e24c00764938?branch=master&workflowId=${{ github.run_id }}&attempt=${{ github.run_attempt }}" target="_blank">Click here!</a> | |
| It seems like there are some failures 😔. We are not able to recognize it, please check this manually <a href="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" target="_blank">here.</a> | |
| - name: Add a comment on the PR when ci-test is success | |
| if: needs.ci-test.result == 'success' && steps.combine_ci.outputs.specs_failed == '0' | |
| uses: peter-evans/create-or-update-comment@v1 | |
| with: | |
| issue-number: ${{ github.event.client_payload.pull_request.number }} | |
| body: | | |
| Workflow run: <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}>. | |
| Commit: `${{ github.event.client_payload.slash_command.args.named.sha }}`. | |
| Cypress dashboard url: <a href="https://internal.appsmith.com/app/cypressdashboard/rundetails-64ec3df0c632e24c00764938?branch=master&workflowId=${{ github.run_id }}&attempt=${{ github.run_attempt }}" target="_blank">Click here!</a> | |
| All cypress tests have passed 🎉🎉🎉 | |
| # Update check run called "ci-test-result" | |
| - name: Mark ci-test-result job as complete | |
| uses: actions/github-script@v6 | |
| id: update-check-run | |
| if: ${{ always() }} | |
| env: | |
| run_id: ${{ github.run_id }} | |
| repository: ${{ github.repository }} | |
| number: ${{ github.event.client_payload.pull_request.number }} | |
| job: ${{ github.job }} | |
| # Conveniently, job.status maps to https://developer.github.com/v3/checks/runs/#update-a-check-run | |
| conclusion: ${{ job.status }} | |
| matrix_result: ${{ toJson(needs.ci-test) }} | |
| with: | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| script: | | |
| const { data: pull } = await github.rest.pulls.get({ | |
| ...context.repo, | |
| pull_number: process.env.number | |
| }); | |
| const ref = pull.head.sha; | |
| const { data: checks } = await github.rest.checks.listForRef({ | |
| ...context.repo, | |
| ref | |
| }); | |
| const check = checks.check_runs.filter(c => c.name === process.env.job); | |
| if(check.length == 0) { | |
| const head_sha = pull.head.sha; | |
| const { data: completed_at } = await github.rest.checks.create({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| head_sha: head_sha, | |
| name: process.env.job, | |
| status: 'completed', | |
| conclusion: JSON.parse(process.env.matrix_result).result, | |
| output: { | |
| title: "Integration tests result for ok to test", | |
| summary: "https://github.com/" + process.env.repository + "/actions/runs/" + process.env.run_id | |
| } | |
| }); | |
| return completed_at; | |
| } else { | |
| try { | |
| const { data: result } = await github.rest.checks.update({ | |
| ...context.repo, | |
| check_run_id: check[0].id, | |
| status: 'completed', | |
| conclusion: JSON.parse(process.env.matrix_result).result, | |
| output: { | |
| title: "Integration tests result for ok to test", | |
| summary: "https://github.com/" + process.env.repository + "/actions/runs/" + process.env.run_id | |
| } | |
| }); | |
| console.log({ result }); | |
| return result; | |
| } catch(e) { | |
| console.error({ error: e.message }); | |
| } | |
| } | |
| - name: Dump the client payload context | |
| env: | |
| PAYLOAD_CONTEXT: ${{ toJson(github.event.client_payload) }} | |
| run: echo "$PAYLOAD_CONTEXT" | |
| - name: Check ci-test set status | |
| if: needs.ci-test.result != 'success' | |
| run: exit 1 | |
| package: | |
| needs: [ci-test] | |
| runs-on: ubuntu-latest | |
| defaults: | |
| run: | |
| working-directory: app/client | |
| # Run this job only if all the previous steps are a success and the reference if the release or master branch | |
| if: success() && (github.ref == 'refs/heads/release' || github.ref == 'refs/heads/master') | |
| steps: | |
| # Update check run called "package" | |
| - name: Mark package job as complete | |
| uses: actions/github-script@v6 | |
| id: update-check-run | |
| if: ${{ always() }} | |
| env: | |
| run_id: ${{ github.run_id }} | |
| repository: ${{ github.repository }} | |
| number: ${{ github.event.client_payload.pull_request.number }} | |
| job: ${{ github.job }} | |
| # Conveniently, job.status maps to https://developer.github.com/v3/checks/runs/#update-a-check-run | |
| conclusion: ${{ job.status }} | |
| with: | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| script: | | |
| const { data: pull } = await github.rest.pulls.get({ | |
| ...context.repo, | |
| pull_number: process.env.number | |
| }); | |
| const ref = pull.head.sha; | |
| const { data: checks } = await github.rest.checks.listForRef({ | |
| ...context.repo, | |
| ref | |
| }); | |
| const check = checks.check_runs.filter(c => c.name === process.env.job); | |
| if(check.length == 0) { | |
| const head_sha = pull.head.sha; | |
| const { data: completed_at } = await github.rest.checks.create({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| head_sha: head_sha, | |
| name: process.env.job, | |
| status: 'completed', | |
| conclusion: process.env.conclusion, | |
| output: { | |
| title: "Package result for ok to test", | |
| summary: "https://github.com/" + process.env.repository + "/actions/runs/" + process.env.run_id | |
| } | |
| }); | |
| return completed_at; | |
| } else { | |
| const { data: result } = await github.rest.checks.update({ | |
| ...context.repo, | |
| check_run_id: check[0].id, | |
| status: 'completed', | |
| conclusion: process.env.conclusion, | |
| output: { | |
| title: "Package result for ok to test", | |
| summary: "https://github.com/" + process.env.repository + "/actions/runs/" + process.env.run_id | |
| } | |
| }); | |
| return result; | |
| } |