update long tests to only use the last two runs for correlation plots (…

…#362)

* update long tests to only use the last two runs for correlation plots

* swap long and short tests

* fix typo

* fix csv paths

* silence wget stderr to cleanup log

.github/workflows/long-tests.yml

@@ -8,7 +8,7 @@ on:
   push:
     branches-ignore:
       - "gh-readonly-queue**"
-  merge_group:
+  pull_request:
 
   # Allows you to run this workflow manually from the Actions tab
   workflow_dispatch:
@@ -55,24 +55,32 @@ jobs:
           ./util/job_launching/get_stats.py -k -K -R -B GPU_Microbenchmark -C RTX2060-SASS -A | tee turing-ubench-sass-$GITHUB_RUN_NUMBER"_"$GITHUB_RUN_ATTEMPT.csv
           ./util/job_launching/get_stats.py -k -K -R -B GPU_Microbenchmark -C RTX3070-SASS -A | tee ampere-ubench-sass-$GITHUB_RUN_NUMBER"_"$GITHUB_RUN_ATTEMPT.csv
           mkdir -p statistics-archive/ubench/
+          # First we merge and archive this run to the main csv that contains all previous runs
           ./util/plotting/merge-stats.py -R -c ./statistics-archive/ubench/v100-ubench-sass.csv,v100-ubench-sass-$GITHUB_RUN_NUMBER"_"$GITHUB_RUN_ATTEMPT.csv \
           | tee v100-ubench-sass.csv && mv v100-ubench-sass.csv ./statistics-archive/ubench/
           ./util/plotting/merge-stats.py -R -c ./statistics-archive/ubench/turing-ubench-sass.csv,turing-ubench-sass-$GITHUB_RUN_NUMBER"_"$GITHUB_RUN_ATTEMPT.csv \
           | tee turing-ubench-sass.csv && mv turing-ubench-sass.csv ./statistics-archive/ubench/
           ./util/plotting/merge-stats.py -R -c ./statistics-archive/ubench/ampere-ubench-sass.csv,ampere-ubench-sass-$GITHUB_RUN_NUMBER"_"$GITHUB_RUN_ATTEMPT.csv \
           | tee ampere-ubench-sass.csv && mv ampere-ubench-sass.csv ./statistics-archive/ubench/
+          # Next we merge the latest run with the current run (used for correlation plots) then archive the current run as the new latest for the next time this action occurs
+          ./util/plotting/merge-stats.py -R -c ./statistics-archive/ubench/v100-ubench-sass-latest.csv,v100-ubench-sass-$GITHUB_RUN_NUMBER"_"$GITHUB_RUN_ATTEMPT.csv \
+          | tee v100-ubench-sass-latest2.csv && mv v100-ubench-sass-$GITHUB_RUN_NUMBER"_"$GITHUB_RUN_ATTEMPT.csv ./statistics-archive/ubench/v100-ubench-sass-latest.csv
+          ./util/plotting/merge-stats.py -R -c ./statistics-archive/ubench/turing-ubench-sass-latest.csv,turing-ubench-sass-$GITHUB_RUN_NUMBER"_"$GITHUB_RUN_ATTEMPT.csv \
+          | tee turing-ubench-sass-latest2.csv && mv turing-ubench-sass-$GITHUB_RUN_NUMBER"_"$GITHUB_RUN_ATTEMPT.csv ./statistics-archive/ubench/turing-ubench-sass-latest.csv
+          ./util/plotting/merge-stats.py -R -c ./statistics-archive/ubench/ampere-ubench-sass-latest.csv,ampere-ubench-sass-$GITHUB_RUN_NUMBER"_"$GITHUB_RUN_ATTEMPT.csv \
+          | tee ampere-ubench-sass-latest2.csv && mv ampere-ubench-sass-$GITHUB_RUN_NUMBER"_"$GITHUB_RUN_ATTEMPT.csv ./statistics-archive/ubench/ampere-ubench-sass-latest.csv
           git -C ./statistics-archive add --all
           git -C ./statistics-archive commit \
           -m "Jenkins automated checkin git_${GITHUB_REF}"_"$GITHUB_RUN_NUMBER"_"$GITHUB_RUN_ATTEMPT Build:$GITHUB_RUN_NUMBER"_"$GITHUB_RUN_ATTEMPT" || echo "No Changes."
           git -C ./statistics-archive push -u origin git_${GITHUB_REF}"_"$GITHUB_RUN_NUMBER"_"$GITHUB_RUN_ATTEMPT
       - name: Correlate Ubench
         run: |
           source ./env-setup/11.7.0_env_setup.sh
-          ./util/hw_stats/get_hw_data.sh > /dev/null
+          ./util/hw_stats/get_hw_data.sh > /dev/null 2>&1
           rm -rf ./util/plotting/correl-html/
-          ./util/plotting/plot-correlation.py -c ./statistics-archive/ubench/v100-ubench-sass.csv -H ./hw_run/QUADRO-V100/device-0/10.2/ | tee v100-ubench-correl.txt
-          ./util/plotting/plot-correlation.py -c ./statistics-archive/ubench/turing-ubench-sass.csv -H ./hw_run/TURING-RTX2060/10.2/ | tee turing-ubench-correl.txt
-          ./util/plotting/plot-correlation.py -c ./statistics-archive/ubench/ampere-ubench-sass.csv -H ./hw_run/AMPERE-RTX3070/11.2/ | tee ampere-ubench-correl.txt
+          ./util/plotting/plot-correlation.py -c ./v100-ubench-sass-latest2.csv -H ./hw_run/QUADRO-V100/device-0/10.2/ | tee v100-ubench-correl.txt
+          ./util/plotting/plot-correlation.py -c ./turing-ubench-sass-latest2.csv -H ./hw_run/TURING-RTX2060/10.2/ | tee turing-ubench-correl.txt
+          ./util/plotting/plot-correlation.py -c ./ampere-ubench-sass-latest2.csv -H ./hw_run/AMPERE-RTX3070/11.2/ | tee ampere-ubench-correl.txt
           ssh ghci@tgrogers-pc01 mkdir -p /home/ghci/accel-sim/correl/git_${GITHUB_REF}"_"$GITHUB_RUN_NUMBER"_"$GITHUB_RUN_ATTEMPT/
           rsync --delete -r ./util/plotting/correl-html/ ghci@tgrogers-pc01:/home/ghci/accel-sim/correl/git_${GITHUB_REF}"_"$GITHUB_RUN_NUMBER"_"$GITHUB_RUN_ATTEMPT/
           BODY="Github CI - Build $GITHUB_REF SUCCESS.

.github/workflows/short-tests.yml

@@ -8,7 +8,7 @@ on:
   push:
     branches-ignore:
       - "gh-readonly-queue**"
-  pull_request:
+  merge_group:
 
   # Allows you to run this workflow manually from the Actions tab
   workflow_dispatch: