Merge pull request #999 from rylev/significance-threshold

Add a severity column

Author: Mark-Simulacrum

site/src/api.rs

@@ -187,6 +187,7 @@ pub mod comparison {
         pub profile: String,
         pub scenario: String,
         pub is_significant: bool,
+        pub significance_threshold: f64,
         pub is_dodgy: bool,
         pub historical_statistics: Option<Vec<f64>>,
         pub statistics: (f64, f64),

site/src/comparison.rs

@@ -114,6 +114,7 @@ pub async fn handle_compare(
             scenario: comparison.scenario.to_string(),
             is_dodgy: comparison.is_dodgy(),
             is_significant: comparison.is_significant(),
+            significance_threshold: comparison.signifcance_threshold() * 100.0,
             historical_statistics: comparison.variance.map(|v| v.data),
             statistics: comparison.results,
         })

site/static/compare.html

@@ -192,8 +192,15 @@
             text-align: center;
         }
 
-        #benches th {
+        #benches tbody:first-child th {
+            text-align: center;
+        }
+
+        #benches tbody:not(:first-child) th {
             border-right: dotted 1px;
+        }
+
+        #benches th {
             text-align: left;
             word-break: break-word;
             width: 25%;
@@ -401,6 +408,26 @@ <h2>Comparing <span id="stat-header">{{stat}}</span> between <span id="before">{
                 </div>
             </div>
             <table id="benches" class="compare">
+                <tbody>
+                    <tr>
+                        <th>Name & Profile</th>
+                        <th>Scenario</th>
+                        <th>{{before}}</th>
+                        <th>{{after}}</th>
+                        <th>% Change</th>
+                        <th>
+                            Significance Factor<span class="tooltip">?
+                                <span class="tooltiptext">
+                                    How much a particular result is over the significance threshold. A factor of 2.50x
+                                    means
+                                    the result is 2.5 times over the significance threshold. You can see <a
+                                        href="https://github.com/rust-lang/rustc-perf/blob/master/docs/comparison-analysis.md#what-makes-a-test-result-significant">
+                                        here</a> how the significance threshold is calculated.
+                                </span>
+                            </span>
+                        </th>
+                    </tr>
+                </tbody>
                 <template v-for="bench in benches">
                     <tbody>
                         <template v-for="run in bench.variants">
@@ -425,6 +452,9 @@ <h2>Comparing <span id="stat-header">{{stat}}</span> between <span id="before">{
                                         </span>
                                     </a>
                                 </td>
+                                <td>
+                                    {{ Math.abs(run.percent / run.significanceThreshold).toFixed(2) }}x
+                                </td>
                             </tr>
                         </template>
                     </tbody>
@@ -541,6 +571,7 @@ <h2>Comparing <span id="stat-header">{{stat}}</span> between <span id="before">{
                                     datumB,
                                     percent,
                                     isDodgy,
+                                    significanceThreshold: r.significance_threshold,
                                     isSignificant
                                 });
                             }