From 0e1bb1f40aeffc236fada503045c761e04f8265c Mon Sep 17 00:00:00 2001 From: Karan Batavia <118820668+karan-batavia@users.noreply.github.com> Date: Mon, 9 Oct 2023 15:45:27 +0530 Subject: [PATCH] parallel processing for comparisom report (#321) --- .github/workflows/comparison-result.yml | 132 ++++++++++++++++++------ 1 file changed, 101 insertions(+), 31 deletions(-) diff --git a/.github/workflows/comparison-result.yml b/.github/workflows/comparison-result.yml index 6fb35ad0..0aa8aa79 100644 --- a/.github/workflows/comparison-result.yml +++ b/.github/workflows/comparison-result.yml @@ -1,15 +1,38 @@ -name: Monitoring Stability and Comparing Results for privado +name: Monitoring Stability and Comparing Results -# Triggers when a pull_request or a push action is configured on master branch +# Triggers when a pull_request is created on: pull_request_target: + branches: + - "**" jobs: + start_workflow: + runs-on: ubuntu-latest + steps: + - name: Send message to slack + id: initial-message + uses: archive/github-actions-slack@master + with: + slack-optional-parse: full + slack-bot-user-oauth-access-token: ${{ secrets.SLACK_TOKEN }} + slack-channel: ${{ secrets.SLACK_CHANNEL_ID }} + slack-text: "Comparison workflow started for ${{github.event.pull_request.html_url}}" + + - name: Save output to env + id: save-output + run: echo "INIT_MSG_TS=${{ fromJson(steps.initial-message.outputs.slack-result).response.message.ts }}" >> $GITHUB_OUTPUT + outputs: + init_message_ts: ${{steps.save-output.outputs.INIT_MSG_TS}} setup_and_scan: + needs: start_workflow + strategy: + matrix: + language: ['java', 'python', 'js', 'ruby-1', 'ruby-2'] + continue-on-error: true runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - name: Install JDK-18 uses: actions/setup-java@v3 with: @@ -26,53 +49,100 @@ jobs: with: python-version: '3.10' - - name: Clone standalone-monitoring-stability/main + - name: Clone standalone-monitoring-stability/flow-test uses: actions/checkout@v3 with: repository: Privado-Inc/standalone-monitoring-stability path: ./temp/standalone-monitoring-stability - ref: main - + ref: main + + # langauge specific repository file - name: Run the script for ${{github.head_ref}} and ${{github.base_ref}} - run: cd ./temp/standalone-monitoring-stability && pip install -r requirements.txt && python3 ./run.py -rbb ${{github.base_ref}} -rbh ${{github.head_ref}} -brr ${{ github.event.pull_request.base.repo.html_url }} -hrr ${{ github.event.pull_request.head.repo.html_url }} -guf -urc + run: cd ./temp/standalone-monitoring-stability && pip install -r requirements.txt && python3 ./run.py -r ./repos/${{matrix.language}}.txt -b ${{github.base_ref}} -h ${{github.head_ref}} -bcr ${{ github.event.pull_request.base.repo.html_url }} -hcr ${{ github.event.pull_request.head.repo.html_url }} -guf - name: Run aws-export - run: cd ./temp/standalone-monitoring-stability/ && python3 aws-export.py ${{github.event.number}} + run: cd ./temp/standalone-monitoring-stability/ && python3 aws-export.py ${{matrix.language}}-${{github.event.number}} - name: Move results to a folder - run: cd ./temp/standalone-monitoring-stability/ && mkdir results && mv output-${{github.event.number}}.xlsx ./results/output-${{github.event.number}}.xlsx && mv ./temp/result-${{github.event.number}}.zip ./results/result-${{github.event.number}}.zip && mv slack_summary.txt ./results/slack_summary.txt + run: cd ./temp/standalone-monitoring-stability/ && mkdir results && mv output-${{matrix.language}}-${{github.event.number}}.xlsx ./results/output-${{matrix.language}}-${{github.event.number}}.xlsx && mv ./temp/result-${{matrix.language}}-${{github.event.number}}.zip ./results/result-${{matrix.language}}-${{github.event.number}}.zip && mv slack_summary.txt ./results/slack_summary.txt + + # Send result to slack + - name: Zip the results + run: zip result-${{matrix.language}}-${{github.event.number}}.zip -r /home/runner/work/privado-core/privado-core/temp/standalone-monitoring-stability/results + + - name: Set summary variable + run: | + echo "MESSAGE<> $GITHUB_ENV + echo "$(cat /home/runner/work/privado-core/privado-core/temp/standalone-monitoring-stability/results/slack_summary.txt)" >> $GITHUB_ENV + echo "EOF" >> $GITHUB_ENV + + - name: Post results to slack + uses: MeilCli/slack-upload-file@v3 + with: + thread_ts: ${{needs.start_workflow.outputs.init_message_ts}} + slack_token: ${{ secrets.SLACK_TOKEN }} + channel_id: ${{ secrets.SLACK_CHANNEL_ID }} + file_path: "/home/runner/work/privado-core/privado-core/result-${{matrix.language}}-${{github.event.number}}.zip" + initial_comment: "Comparison Results generated on ${{github.event.repository.name}} by PR ${{github.event.number}} from branch ${{github.head_ref}} to ${{github.base_ref}} \nPR link https://github.com/Privado-Inc/privado-core/pull/${{github.event.number}}\n Language: ${{matrix.language}} \nSummary Report:\n ${{ env.MESSAGE }}" + file_type: "zip" + + - name: Export workflow output + run: cd ./temp/standalone-monitoring-stability && python3 ./workflow_check.py /home/runner/work/privado-core/privado-core/temp/standalone-monitoring-stability/results/slack_summary.txt + + - name: Set summary variable + run: | + echo "MESSAGE<> $GITHUB_ENV + echo "$(cat ./temp/standalone-monitoring-stability/action_result.txt)" >> $GITHUB_ENV + echo "EOF" >> $GITHUB_ENV - - name: Upload output and result for next job + - name: Print action result + run: cat ./temp/standalone-monitoring-stability/action_result.txt + + - name: Upload summary file uses: actions/upload-artifact@master with: - name: results - path: /home/runner/work/privado/privado/temp/standalone-monitoring-stability/results + name: ${{matrix.language}} + path: /home/runner/work/privado-core/privado-core/temp/standalone-monitoring-stability/results/slack_summary.txt - send-result: - needs: setup_and_scan - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 + - name: Workflow report analysis + if: ${{ env.MESSAGE != 'true' }} + run: exit 1 - - name: Download result folder + collate_summary: + needs: [start_workflow, setup_and_scan] + runs-on: ubuntu-latest + steps: + - name: Download summary file uses: actions/download-artifact@master with: - name: results - path: ./results - - - name: Zip the results - run: zip result-${{github.event.number}}.zip -r ./results + path: ./language_summary + + - name: Install Python 3.10 + uses: actions/setup-python@v4 + with: + python-version: '3.10' + + - name: Clone standalone-monitoring-stability/flow-test + uses: actions/checkout@v3 + with: + repository: Privado-Inc/standalone-monitoring-stability + path: ./temp/standalone-monitoring-stability + ref: main + + - name: Collate summary + run: cd ./temp/standalone-monitoring-stability && pip install -r requirements.txt && python3 ./collate_summary.py -s /home/runner/work/privado-core/privado-core/language_summary - name: Set summary variable run: | echo "MESSAGE<> $GITHUB_ENV - echo "$(cat ./results/slack_summary.txt)" >> $GITHUB_ENV + echo "$(cat /home/runner/work/privado-core/privado-core/temp/standalone-monitoring-stability/global_summary.txt)" >> $GITHUB_ENV echo "EOF" >> $GITHUB_ENV - - name: Post results to slack - uses: MeilCli/slack-upload-file@v3 + + - name: Send summary to slack + uses: slackapi/slack-github-action@v1.24.0 with: - slack_token: ${{ secrets.SLACK_TOKEN }} - channel_id: ${{ secrets.SLACK_CHANNEL_ID }} - file_path: "/home/runner/work/privado/privado/result-${{github.event.number}}.zip" - initial_comment: "Comparison Results generated on ${{github.event.repository.name}} by PR ${{github.event.number}} from branch ${{github.head_ref}} to ${{github.base_ref}} \nPR link https://github.com/Privado-Inc/privado/pull/${{github.event.number}} \nSummary Report:\n ${{ env.MESSAGE }}" - file_type: "zip" \ No newline at end of file + update-ts: ${{needs.start_workflow.outputs.init_message_ts}} + channel-id: ${{ secrets.SLACK_CHANNEL_ID }} + slack-message: "\nComparison Results generated on ${{github.event.repository.name}} by PR ${{github.event.number}} from branch ${{github.head_ref}} to ${{github.base_ref}} \nPR link https://github.com/Privado-Inc/privado-core/pull/${{github.event.number}}\nLanguage: All \nSummary Report:\n ${{ env.MESSAGE }}" + env: + SLACK_BOT_TOKEN: ${{ secrets.SLACK_TOKEN }}