From edcc2004268148c1beae27e3d27aec8112949a13 Mon Sep 17 00:00:00 2001 From: Karan Batavia <118820668+karan-batavia@users.noreply.github.com> Date: Mon, 9 Oct 2023 19:05:06 +0530 Subject: [PATCH] parallel processing for comparison report (#324) --- .github/workflows/comparison-result.yml | 34 ++++++++++++------------- 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/.github/workflows/comparison-result.yml b/.github/workflows/comparison-result.yml index 0aa8aa79..76dce609 100644 --- a/.github/workflows/comparison-result.yml +++ b/.github/workflows/comparison-result.yml @@ -1,4 +1,4 @@ -name: Monitoring Stability and Comparing Results +name: Monitoring Stability and Comparing Results for privado # Triggers when a pull_request is created on: @@ -24,6 +24,7 @@ jobs: run: echo "INIT_MSG_TS=${{ fromJson(steps.initial-message.outputs.slack-result).response.message.ts }}" >> $GITHUB_OUTPUT outputs: init_message_ts: ${{steps.save-output.outputs.INIT_MSG_TS}} + setup_and_scan: needs: start_workflow strategy: @@ -33,6 +34,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 + - name: Install JDK-18 uses: actions/setup-java@v3 with: @@ -49,16 +51,15 @@ jobs: with: python-version: '3.10' - - name: Clone standalone-monitoring-stability/flow-test + - name: Clone standalone-monitoring-stability/main uses: actions/checkout@v3 with: repository: Privado-Inc/standalone-monitoring-stability path: ./temp/standalone-monitoring-stability ref: main - # langauge specific repository file - name: Run the script for ${{github.head_ref}} and ${{github.base_ref}} - run: cd ./temp/standalone-monitoring-stability && pip install -r requirements.txt && python3 ./run.py -r ./repos/${{matrix.language}}.txt -b ${{github.base_ref}} -h ${{github.head_ref}} -bcr ${{ github.event.pull_request.base.repo.html_url }} -hcr ${{ github.event.pull_request.head.repo.html_url }} -guf + run: cd ./temp/standalone-monitoring-stability && pip install -r requirements.txt && python3 ./run.py -r ./repos/${{matrix.language}}.txt -rbb ${{github.base_ref}} -rbh ${{github.head_ref}} -brr ${{ github.event.pull_request.base.repo.html_url }} -hrr ${{ github.event.pull_request.head.repo.html_url }} -guf -urc - name: Run aws-export run: cd ./temp/standalone-monitoring-stability/ && python3 aws-export.py ${{matrix.language}}-${{github.event.number}} @@ -66,14 +67,13 @@ jobs: - name: Move results to a folder run: cd ./temp/standalone-monitoring-stability/ && mkdir results && mv output-${{matrix.language}}-${{github.event.number}}.xlsx ./results/output-${{matrix.language}}-${{github.event.number}}.xlsx && mv ./temp/result-${{matrix.language}}-${{github.event.number}}.zip ./results/result-${{matrix.language}}-${{github.event.number}}.zip && mv slack_summary.txt ./results/slack_summary.txt - # Send result to slack - name: Zip the results - run: zip result-${{matrix.language}}-${{github.event.number}}.zip -r /home/runner/work/privado-core/privado-core/temp/standalone-monitoring-stability/results + run: zip result-${{matrix.language}}-${{github.event.number}}.zip -r /home/runner/work/privado/privado/temp/standalone-monitoring-stability/results - name: Set summary variable run: | echo "MESSAGE<> $GITHUB_ENV - echo "$(cat /home/runner/work/privado-core/privado-core/temp/standalone-monitoring-stability/results/slack_summary.txt)" >> $GITHUB_ENV + echo "$(cat /home/runner/work/privado/privado/temp/standalone-monitoring-stability/results/slack_summary.txt)" >> $GITHUB_ENV echo "EOF" >> $GITHUB_ENV - name: Post results to slack @@ -82,12 +82,12 @@ jobs: thread_ts: ${{needs.start_workflow.outputs.init_message_ts}} slack_token: ${{ secrets.SLACK_TOKEN }} channel_id: ${{ secrets.SLACK_CHANNEL_ID }} - file_path: "/home/runner/work/privado-core/privado-core/result-${{matrix.language}}-${{github.event.number}}.zip" - initial_comment: "Comparison Results generated on ${{github.event.repository.name}} by PR ${{github.event.number}} from branch ${{github.head_ref}} to ${{github.base_ref}} \nPR link https://github.com/Privado-Inc/privado-core/pull/${{github.event.number}}\n Language: ${{matrix.language}} \nSummary Report:\n ${{ env.MESSAGE }}" + file_path: "/home/runner/work/privado/privado/result-${{matrix.language}}-${{github.event.number}}.zip" + initial_comment: "Comparison Results generated on ${{github.event.repository.name}} by PR ${{github.event.number}} from branch ${{github.head_ref}} to ${{github.base_ref}} \nPR link https://github.com/Privado-Inc/privado/pull/${{github.event.number}}\n Language: ${{matrix.language}} \nSummary Report:\n ${{ env.MESSAGE }}" file_type: "zip" - name: Export workflow output - run: cd ./temp/standalone-monitoring-stability && python3 ./workflow_check.py /home/runner/work/privado-core/privado-core/temp/standalone-monitoring-stability/results/slack_summary.txt + run: cd ./temp/standalone-monitoring-stability && python3 ./workflow_check.py /home/runner/work/privado/privado/temp/standalone-monitoring-stability/results/slack_summary.txt - name: Set summary variable run: | @@ -95,14 +95,11 @@ jobs: echo "$(cat ./temp/standalone-monitoring-stability/action_result.txt)" >> $GITHUB_ENV echo "EOF" >> $GITHUB_ENV - - name: Print action result - run: cat ./temp/standalone-monitoring-stability/action_result.txt - - name: Upload summary file uses: actions/upload-artifact@master with: name: ${{matrix.language}} - path: /home/runner/work/privado-core/privado-core/temp/standalone-monitoring-stability/results/slack_summary.txt + path: /home/runner/work/privado/privado/temp/standalone-monitoring-stability/results/slack_summary.txt - name: Workflow report analysis if: ${{ env.MESSAGE != 'true' }} @@ -129,13 +126,14 @@ jobs: path: ./temp/standalone-monitoring-stability ref: main + - name: Collate summary - run: cd ./temp/standalone-monitoring-stability && pip install -r requirements.txt && python3 ./collate_summary.py -s /home/runner/work/privado-core/privado-core/language_summary + run: cd ./temp/standalone-monitoring-stability && pip install -r requirements.txt && python3 ./collate_summary.py -s /home/runner/work/privado/privado/language_summary - name: Set summary variable run: | echo "MESSAGE<> $GITHUB_ENV - echo "$(cat /home/runner/work/privado-core/privado-core/temp/standalone-monitoring-stability/global_summary.txt)" >> $GITHUB_ENV + echo "$(cat /home/runner/work/privado/privado/temp/standalone-monitoring-stability/global_summary.txt)" >> $GITHUB_ENV echo "EOF" >> $GITHUB_ENV - name: Send summary to slack @@ -143,6 +141,6 @@ jobs: with: update-ts: ${{needs.start_workflow.outputs.init_message_ts}} channel-id: ${{ secrets.SLACK_CHANNEL_ID }} - slack-message: "\nComparison Results generated on ${{github.event.repository.name}} by PR ${{github.event.number}} from branch ${{github.head_ref}} to ${{github.base_ref}} \nPR link https://github.com/Privado-Inc/privado-core/pull/${{github.event.number}}\nLanguage: All \nSummary Report:\n ${{ env.MESSAGE }}" + slack-message: "\nComparison Results generated on ${{github.event.repository.name}} by PR ${{github.event.number}} from branch ${{github.head_ref}} to ${{github.base_ref}} \nPR link https://github.com/Privado-Inc/privado/pull/${{github.event.number}}\nLanguage: All \nSummary Report:\n ${{ env.MESSAGE }}" env: - SLACK_BOT_TOKEN: ${{ secrets.SLACK_TOKEN }} + SLACK_BOT_TOKEN: ${{ secrets.SLACK_TOKEN }} \ No newline at end of file