Replace p90 and P2Estimator with Simple Exponential Smoother (SES) (#867)

* Replace p2 estimator with SES

* Fix PID tests to use new smoother

* Remove p2

* update images

* Correct comment

* update images

* Clean up comments, address Abed's concerns

* Update comment

* Update images

* run all experiments

* Unit tests for success criteria

* Add incident detection and adaptive convergence

* Update success criteria version 2

* Address Abed's comments

* Update experiments Gemfile to prevent error

* Update experiments

* Address Abed comments again

* Remove init

* Experiments

---------

Co-authored-by: Abdulrahman Alhamali <abdulrahman.alhamali@shopify.com>

Author: kris-gaudel

experiments/Gemfile

@@ -7,3 +7,4 @@ gemspec path: ".."
 # Graphing library for visualization
 gem "gruff", "~> 0.23"
 gem "logger"
+gem "csv"

experiments/Gemfile.lock

@@ -5,6 +5,7 @@ module Experiments
     # Experiment runner for circuit breaker experiments (both adaptive and classic)
     # Handles all the common logic: service creation, threading, monitoring, analysis, and visualization
     require "fileutils"
+    require "csv"
     class DegradationPhase
       attr_reader :healthy, :error_rate, :latency, :specific_endpoint_latency
 
@@ -680,8 +681,6 @@ def add_state_transition_markers(graph, bucketed_data, bucket_size, num_buckets)
       end
 
       def save_csv_outputs
-        require "csv"
-
         # Save time-based analysis CSV
         if @time_analysis_data.any?
           csv_path = File.join(@csv_results_path, @time_analysis_csv_filename)

experiments/experiment_helpers.rb

@@ -1,60 +1,61 @@
 Window,Thread Count,Error Rate Avg,Error Rate Min,Error Rate Max,Ideal Error Rate,Rejection Rate Avg,Rejection Rate Min,Rejection Rate Max,Integral Avg,Integral Min,Integral Max,Derivative Avg,Derivative Min,Derivative Max,Total Request Time
-1,60,1.05,0.0,4.0,1.0,0.36,0.0,2.7,0.0399,0.0,0.3,0.0,-0.0005,0.0015,645.54
-2,60,0.89,0.0,3.96,1.0,0.29,0.0,2.71,0.0558,0.0,0.3,-0.0003,-0.0034,0.002,1244.97
-3,60,1.07,0.0,3.96,1.0,0.46,0.0,2.72,0.0822,-0.0039,0.402,0.0001,-0.002,0.0024,1844.98
-4,60,0.91,0.0,4.9,1.0,0.38,0.0,3.51,0.0825,-0.0045,0.3847,-0.0002,-0.0019,0.002,2444.77
-5,60,1.01,0.0,2.97,1.02,0.33,0.0,1.95,0.0785,-0.032,0.3639,0.0001,-0.0037,0.0015,3045.41
-6,60,1.11,0.0,3.03,1.04,0.43,0.0,2.04,0.0907,-0.032,0.3639,0.0001,-0.0023,0.002,3645.04
-7,60,1.66,0.0,5.94,1.04,0.81,0.0,4.53,0.1318,-0.03,0.5669,0.0002,-0.0019,0.0025,4245.33
-8,60,1.38,0.0,3.96,1.06,0.69,0.0,2.7,0.1351,-0.0346,0.4502,-0.0003,-0.0047,0.002,4845.24
-9,60,1.49,0.0,4.95,1.1,0.77,0.0,3.78,0.1589,-0.0346,0.5467,0.0001,-0.0032,0.0028,5444.28
-10,60,1.67,0.0,4.9,1.1,0.87,0.0,3.97,0.1678,-0.0231,0.4997,0.0,-0.003,0.0024,6044.61
-11,60,1.62,0.0,4.0,1.17,0.88,0.0,2.95,0.1811,-0.0421,0.5309,-0.0001,-0.0037,0.0026,6643.87
-12,60,1.25,0.0,3.96,1.17,0.58,0.0,2.79,0.1562,-0.0486,0.5309,-0.0002,-0.0032,0.0029,7244.52
-13,60,1.79,0.0,5.88,1.17,0.94,0.0,4.55,0.1929,-0.0134,0.6574,0.0004,-0.0023,0.0029,7845.15
-14,60,1.78,0.0,5.88,1.28,0.98,0.0,4.57,0.196,-0.0504,0.6105,-0.0002,-0.0036,0.0029,8444.55
-15,60,2.22,0.0,7.77,1.28,1.3,0.0,6.26,0.2247,-0.0839,0.7227,0.0002,-0.0033,0.0041,9044.39
-16,60,2.28,0.0,5.94,1.43,1.29,0.0,4.91,0.2141,-0.0669,0.6019,-0.0002,-0.0051,0.0028,9645.25
-17,60,1.9,0.0,4.95,1.43,0.96,0.0,3.75,0.1924,-0.0646,0.5866,-0.0002,-0.0038,0.003,10244.44
-18,60,2.49,0.0,6.86,1.61,1.41,0.0,5.1,0.2453,-0.0707,0.6874,0.0004,-0.0029,0.004,10844.63
-19,60,2.36,0.0,5.94,1.82,1.24,0.0,4.58,0.2392,-0.1201,0.6874,-0.0003,-0.0051,0.0028,11444.13
-20,60,2.25,0.0,5.83,2.01,1.03,0.0,4.03,0.21,-0.1201,0.7549,-0.0,-0.0042,0.003,12044.57
-21,60,2.3,0.0,5.88,2.01,1.02,0.0,4.25,0.2143,-0.1201,0.7693,0.0001,-0.004,0.0038,12644.19
-22,60,2.24,0.0,4.95,2.01,0.98,0.0,3.29,0.2116,-0.1201,0.6326,-0.0,-0.004,0.0026,13244.68
-23,60,2.45,0.0,6.8,2.01,1.13,0.0,4.52,0.2328,-0.1201,0.6889,0.0001,-0.003,0.0028,13844.24
-24,60,2.07,0.0,5.88,2.01,0.89,0.0,4.0,0.2276,-0.1201,0.6434,-0.0003,-0.0042,0.0024,14444.05
-25,60,2.75,0.0,6.93,2.18,1.27,0.0,5.0,0.2719,-0.1201,0.8048,0.0004,-0.0035,0.0044,15044.41
-26,60,2.68,0.0,6.86,2.36,1.21,0.0,4.79,0.2554,-0.1201,0.6208,-0.0003,-0.0041,0.0028,15644.62
-27,60,3.29,0.0,7.69,2.56,1.59,0.0,5.65,0.289,-0.0685,0.75,0.0003,-0.0032,0.0034,16244.4
-28,60,3.12,0.0,7.77,2.77,1.44,0.0,4.88,0.2751,-0.1667,0.7015,-0.0003,-0.0057,0.003,16843.99
-29,60,2.69,0.0,7.77,2.95,1.01,0.0,5.1,0.2277,-0.1667,0.922,-0.0002,-0.0035,0.0042,17444.28
-30,60,2.76,0.0,6.8,3.11,0.97,0.0,4.3,0.2378,-0.094,0.7934,0.0002,-0.0045,0.0044,18044.56
-31,60,3.53,0.0,8.74,3.11,1.45,0.0,6.04,0.2862,0.0055,1.0142,0.0003,-0.005,0.0056,18644.9
-32,60,3.33,0.0,8.65,3.11,1.34,0.0,6.23,0.2992,0.0055,0.804,-0.0003,-0.0054,0.0035,19244.11
-33,60,2.94,0.0,7.77,1.72,1.09,0.0,5.06,0.2857,-0.0007,1.2569,-0.0002,-0.0033,0.0041,19843.78
-34,60,3.29,0.0,6.86,1.72,1.25,0.0,4.96,0.2934,-0.1048,0.9277,0.0003,-0.004,0.0041,20443.83
-35,60,2.96,0.0,7.69,1.85,1.04,0.0,5.03,0.2887,-0.0328,0.7995,-0.0003,-0.0042,0.0045,21043.87
-36,60,3.3,0.0,7.77,2.0,1.12,0.0,4.65,0.3086,-0.0796,0.8072,0.0002,-0.005,0.0038,21643.65
-37,60,4.11,0.0,8.65,2.15,1.66,0.0,5.09,0.3542,-0.0397,1.0966,0.0003,-0.0045,0.0043,22243.52
-38,60,3.97,0.0,10.48,2.29,1.62,0.0,6.7,0.3525,-0.1022,0.9956,-0.0004,-0.0054,0.0036,22843.2
-39,60,3.97,1.01,8.65,2.46,1.49,0.0,6.25,0.3482,-0.1022,0.9956,-0.0,-0.0062,0.004,23444.03
-40,60,4.09,0.0,8.65,2.63,1.61,0.0,5.47,0.3569,-0.1346,1.0546,0.0001,-0.0045,0.0046,24043.84
-41,60,4.3,1.02,10.48,2.82,1.72,0.0,6.76,0.3548,-0.1346,1.0055,-0.0,-0.0049,0.0049,24643.78
-42,60,3.82,0.0,8.65,3.77,1.28,0.0,5.81,0.3097,-0.1022,0.9895,-0.0003,-0.0054,0.0035,25243.61
-43,60,4.26,1.01,9.62,3.77,1.49,0.0,6.1,0.3266,-0.0823,1.4515,0.0004,-0.0051,0.0045,25843.32
-44,60,4.55,1.03,9.71,3.15,1.61,0.0,5.98,0.3158,-0.1477,1.1042,-0.0,-0.0048,0.0041,26443.56
-45,60,4.55,1.02,8.74,3.32,1.54,0.0,5.41,0.303,-0.1477,1.0378,-0.0001,-0.0063,0.005,27043.27
-46,60,4.95,1.02,10.48,3.97,1.79,0.0,6.9,0.3218,-0.1356,0.9369,0.0002,-0.0044,0.0038,27643.11
-47,60,4.63,1.02,12.26,4.09,1.57,0.0,7.91,0.3244,-0.1356,0.9953,-0.0003,-0.0052,0.0037,28242.94
-48,60,4.47,0.0,8.65,4.22,1.38,0.0,5.24,0.3193,-0.1628,0.9724,-0.0,-0.0078,0.005,28843.22
-49,60,4.51,0.0,7.77,4.22,1.29,0.0,4.56,0.3062,-0.1628,0.8046,0.0001,-0.0054,0.0041,29443.25
-50,60,5.2,1.01,10.48,4.36,1.71,0.0,6.3,0.3469,-0.1099,1.1212,0.0003,-0.0036,0.0056,30042.65
-51,60,4.84,1.02,8.65,4.55,1.4,0.0,4.92,0.3301,-0.097,0.9085,-0.0004,-0.0069,0.0038,30643.44
-52,60,5.09,1.02,12.15,4.36,1.6,0.0,7.04,0.3556,-0.097,1.0152,0.0003,-0.0047,0.0039,31243.09
-53,60,5.15,2.0,13.08,4.36,1.55,0.0,8.89,0.3664,-0.1346,1.0907,-0.0001,-0.0047,0.005,31842.29
-54,60,4.91,1.02,11.43,4.49,1.44,0.0,6.75,0.3575,-0.096,1.0907,-0.0001,-0.0088,0.0059,32442.19
-55,60,1.15,0.0,5.0,4.49,0.01,0.0,0.63,0.3553,-0.096,1.0907,-0.0018,-0.0086,0.0034,33042.11
-56,60,0.83,0.0,3.03,4.49,0.0,0.0,0.15,0.3528,-0.096,1.0907,0.0006,-0.0013,0.0039,33641.97
-57,60,0.95,0.0,4.0,4.02,0.01,0.0,0.5,0.3515,-0.096,1.0907,0.0001,-0.0016,0.002,34242.48
-58,60,1.13,0.0,4.0,4.49,0.01,0.0,0.44,0.3515,-0.096,1.0907,0.0001,-0.0017,0.002,34842.4
-59,60,1.06,0.0,4.0,4.49,0.0,0.0,0.0,0.3515,-0.096,1.0907,-0.0,-0.0015,0.002,35442.52
+1,60,1.15,0.0,5.0,0.91,0.46,0.0,3.54,0.051,0.0,0.3932,0.0001,-0.0005,0.002,608.43
+2,60,1.11,0.0,4.76,0.91,0.54,0.0,3.51,0.0799,0.0,0.3932,-0.0002,-0.0042,0.0013,1208.1
+3,60,1.01,0.0,3.85,0.96,0.51,0.0,2.72,0.104,0.0,0.3932,-0.0001,-0.0034,0.0019,1807.55
+4,60,1.16,0.0,3.7,0.94,0.64,0.0,2.54,0.1327,0.0,0.4178,0.0001,-0.003,0.0024,2407.14
+5,60,1.08,0.0,3.8,0.85,0.63,0.0,2.66,0.1542,0.0,0.4966,-0.0001,-0.0031,0.0022,3008.17
+6,60,0.81,0.0,3.8,0.86,0.46,0.0,3.09,0.159,-0.0194,0.4634,-0.0001,-0.0028,0.0016,3607.76
+7,60,1.35,0.0,5.19,0.89,0.9,0.0,4.14,0.2076,-0.0668,0.7948,0.0004,-0.0017,0.0034,4208.11
+8,60,1.39,0.0,6.15,0.89,0.95,0.0,5.0,0.2224,-0.1035,0.772,-0.0002,-0.0037,0.003,4807.4
+9,60,1.38,0.0,5.49,0.81,0.97,0.0,4.88,0.2397,-0.1035,0.6266,-0.0,-0.005,0.0025,5408.4
+10,60,1.66,0.0,4.88,0.84,1.25,0.0,4.06,0.287,0.0318,0.6827,0.0001,-0.0033,0.0035,6009.22
+11,60,1.3,0.0,4.35,0.76,0.97,0.0,3.3,0.2805,0.0011,0.6094,-0.0003,-0.0041,0.0028,6607.88
+12,60,1.23,0.0,4.88,0.68,0.93,0.0,3.96,0.2787,-0.0291,0.5581,0.0001,-0.0035,0.004,7208.1
+13,60,2.06,0.0,6.98,0.62,1.62,0.0,6.04,0.3424,-0.0462,0.9657,0.0004,-0.0037,0.0034,7807.69
+14,60,2.04,0.0,5.88,0.63,1.64,0.0,4.67,0.3236,-0.052,0.7201,-0.0004,-0.0059,0.0032,8408.29
+15,60,1.76,0.0,5.41,0.57,1.44,0.0,4.58,0.2911,-0.038,0.7666,-0.0002,-0.0039,0.0036,9008.28
+16,60,1.96,0.0,5.41,0.79,1.58,0.0,4.61,0.2963,-0.0289,0.7333,0.0002,-0.0042,0.0029,9608.08
+17,60,1.97,0.0,5.19,0.66,1.59,0.0,4.24,0.2901,-0.0289,0.6453,-0.0001,-0.004,0.0028,10208.63
+18,60,2.09,0.0,6.1,0.67,1.66,0.0,5.2,0.288,-0.0111,0.9271,0.0001,-0.0037,0.0046,10807.15
+19,60,2.43,0.0,6.85,0.9,1.95,0.0,5.91,0.3116,-0.0238,0.735,0.0001,-0.0039,0.0044,11407.57
+20,60,2.46,0.0,6.67,0.98,2.02,0.0,5.17,0.325,-0.0129,0.7353,-0.0001,-0.0043,0.0045,12008.23
+21,60,2.28,0.0,6.58,0.99,1.86,0.0,5.72,0.2919,-0.0725,0.8516,-0.0001,-0.0047,0.0037,12608.14
+22,60,2.35,0.0,7.23,0.96,1.89,0.0,6.44,0.2857,-0.1347,0.9045,0.0001,-0.0061,0.0039,13206.52
+23,60,2.7,0.0,7.23,1.11,2.15,0.0,5.82,0.2849,-0.1498,0.7963,0.0001,-0.0041,0.0035,13807.84
+24,60,2.64,0.0,8.14,1.17,2.11,0.0,6.29,0.2866,-0.0669,0.8373,-0.0002,-0.0036,0.0031,14407.52
+25,60,2.83,0.0,7.95,1.19,2.27,0.0,6.88,0.2926,-0.1078,0.9933,0.0001,-0.0053,0.0052,15008.63
+26,60,3.01,0.0,9.78,0.96,2.42,0.0,7.76,0.2959,-0.1293,0.9722,0.0,-0.0063,0.0053,15607.64
+27,60,3.28,0.0,7.59,1.05,2.64,0.0,6.8,0.2994,-0.0858,0.9526,0.0,-0.0079,0.0039,16207.93
+28,60,3.14,0.0,7.23,1.05,2.49,0.0,6.53,0.2538,-0.1087,0.862,-0.0002,-0.0048,0.0071,16806.88
+29,60,2.81,0.0,7.14,0.95,2.16,0.0,5.56,0.2147,-0.1251,0.7833,-0.0001,-0.0046,0.0038,17408.37
+30,60,3.53,0.0,7.14,1.0,2.7,0.0,5.84,0.2688,-0.2656,0.9163,0.0005,-0.0055,0.0042,18007.31
+31,60,3.07,0.0,8.24,1.06,2.33,0.0,6.77,0.2156,-0.2154,0.6836,-0.0005,-0.0047,0.0037,18607.66
+32,60,3.41,0.0,8.54,1.08,2.53,0.0,6.75,0.216,-0.1798,0.9068,0.0003,-0.0051,0.0056,19207.32
+33,60,3.45,0.0,8.54,1.09,2.55,0.0,7.52,0.2085,-0.2145,0.7298,-0.0001,-0.005,0.004,19805.94
+34,60,3.74,0.0,10.34,1.11,2.78,0.0,9.07,0.213,-0.211,1.147,0.0001,-0.0053,0.0051,20407.93
+35,60,3.73,0.0,8.99,1.23,2.75,0.0,7.56,0.1969,-0.1927,0.8806,-0.0001,-0.007,0.0053,21007.23
+36,60,2.81,0.0,7.78,1.25,1.96,0.0,6.81,0.1156,-0.2494,0.756,-0.0004,-0.0054,0.0044,21608.42
+37,60,3.58,0.0,9.64,1.35,2.49,0.0,8.56,0.164,-0.2751,0.9296,0.0008,-0.0047,0.0048,22206.28
+38,60,3.51,0.0,12.5,1.39,2.41,0.0,10.2,0.1715,-0.3184,1.129,-0.0003,-0.0053,0.0063,22808.14
+39,60,4.02,0.0,9.41,1.46,2.83,0.0,7.56,0.2058,-0.2584,0.7405,0.0003,-0.0102,0.0041,23408.03
+40,60,3.81,0.0,9.09,1.44,2.68,0.0,7.33,0.1912,-0.2616,0.8804,-0.0003,-0.006,0.0057,24008.47
+41,60,3.85,0.0,9.64,1.54,2.68,0.0,7.45,0.1704,-0.314,0.743,0.0001,-0.0058,0.0057,24607.61
+42,60,4.22,0.0,9.41,1.61,2.94,0.0,7.09,0.1836,-0.2104,0.824,0.0002,-0.006,0.0051,25208.44
+43,60,4.77,1.2,12.79,1.68,3.4,0.0,10.53,0.2196,-0.2209,0.9607,0.0001,-0.0058,0.0046,25806.83
+44,60,4.38,0.0,8.14,1.79,3.09,0.0,6.61,0.1672,-0.2456,0.6874,-0.0004,-0.0076,0.0045,26407.99
+45,60,4.73,0.0,10.0,1.86,3.32,0.0,8.18,0.189,-0.2623,0.8466,0.0003,-0.0041,0.0056,27006.0
+46,60,4.5,0.0,14.46,1.92,3.09,0.0,12.5,0.1531,-0.2962,1.0917,-0.0002,-0.0054,0.0042,27605.49
+47,60,4.25,0.0,10.71,1.98,2.89,0.0,8.42,0.1433,-0.2712,0.8898,-0.0,-0.0091,0.0053,28207.03
+48,60,3.93,0.0,10.0,2.08,2.51,0.0,8.39,0.0977,-0.3188,0.773,-0.0001,-0.0084,0.0063,28807.67
+49,60,4.77,0.0,11.76,2.11,3.13,0.0,8.8,0.1634,-0.4134,1.3621,0.0006,-0.0073,0.0076,29405.61
+50,60,5.27,0.0,11.36,2.2,3.61,0.0,9.39,0.2268,-0.3567,1.0941,-0.0001,-0.0082,0.006,30006.51
+51,60,5.25,0.0,10.53,2.12,3.67,0.0,9.02,0.2268,-0.3007,1.0931,-0.0003,-0.0062,0.005,30606.63
+52,60,4.8,0.0,11.84,1.91,3.28,0.0,9.65,0.1874,-0.3058,1.0694,-0.0003,-0.0069,0.0062,31205.35
+53,60,5.02,0.0,12.5,1.85,3.4,0.0,11.01,0.1887,-0.2873,1.0977,0.0003,-0.0073,0.0066,31808.4
+54,60,5.06,1.2,12.68,1.9,3.33,0.0,10.09,0.1648,-0.4021,1.093,-0.0001,-0.0071,0.005,32406.87
+55,60,0.96,0.0,3.95,1.91,0.36,0.0,2.31,0.033,-0.4601,0.6529,-0.002,-0.0093,0.0033,33005.76
+56,60,0.92,0.0,3.9,1.87,0.21,0.0,2.31,0.0349,-0.4601,0.6529,0.0015,-0.0021,0.0046,33605.86
+57,60,0.9,0.0,3.75,1.79,0.11,0.0,1.73,0.0387,-0.4021,0.6323,0.0001,-0.003,0.0019,34206.41
+58,60,0.72,0.0,3.61,1.75,0.14,0.0,2.55,0.0454,-0.4021,0.7835,0.0,-0.002,0.0013,34805.54
+59,60,0.76,0.0,2.78,1.63,0.08,0.0,1.55,0.0409,-0.4021,0.5135,0.0,-0.0021,0.001,35406.69
+60,60,0.83,0.0,2.7,1.65,0.17,0.0,1.95,0.0505,-0.4021,0.5233,0.0001,-0.001,0.0013,36001.33

experiments/results/csv/gradual_increase_adaptive_pid_controller.csv

@@ -1,11 +1,11 @@
 Time Range,Total Requests,Success,Errors,Error %,Rejected,Rejected %,Target Error Rate %
-0-60s,35494,35032,357,1.01,105,0.3,1.0
-60-120s,35792,34983,540,1.51,269,0.75,1.5
-120-180s,36059,34981,745,2.07,333,0.92,2.0
-180-240s,36161,34906,822,2.27,433,1.2,2.5
-240-300s,36288,34803,1038,2.86,447,1.23,3.0
-300-360s,36394,34810,1164,3.2,420,1.15,3.5
-360-420s,36731,34695,1474,4.01,562,1.53,4.0
-420-480s,36784,34539,1660,4.51,585,1.59,4.5
-480-540s,36949,34569,1815,4.91,565,1.53,5.0
-540-600s,35339,34888,358,1.01,93,0.26,1.0
+0-60s,27869,27447,297,1.07,125,0.45,1.0
+60-120s,28462,27812,394,1.38,256,0.9,1.5
+120-180s,28751,27771,563,1.96,417,1.45,2.0
+180-240s,28880,27596,707,2.45,577,2.0,2.5
+240-300s,29302,27713,885,3.02,704,2.4,3.0
+300-360s,29333,27600,982,3.35,751,2.56,3.5
+360-420s,29267,27475,1097,3.75,695,2.37,4.0
+420-480s,29655,27451,1284,4.33,920,3.1,4.5
+480-540s,29039,26614,1427,4.91,998,3.44,5.0
+540-600s,27721,27236,248,0.89,237,0.85,1.0

experiments/results/csv/gradual_increase_adaptive_time_analysis.csv

@@ -1,11 +1,11 @@
 Time Range,Total Requests,Success,Errors,Error %,Rejected,Rejected %,Target Error Rate %
-0-60s,34553,34553,0,0.0,0,0.0,0.0
-60-120s,235115,21714,312,0.13,213089,90.63,1.5
-120-180s,135502,16346,350,0.26,118806,87.68,2.0
-180-240s,136698,14526,383,0.28,121789,89.09,2.5
-240-300s,134974,12471,403,0.3,122100,90.46,3.0
-300-360s,133924,11366,428,0.32,122130,91.19,3.5
-360-420s,134599,11156,434,0.32,123009,91.39,4.0
-420-480s,131674,9726,458,0.35,121490,92.27,4.5
-480-540s,131649,8707,473,0.36,122469,93.03,5.0
-540-600s,60201,30326,6,0.01,29869,49.62,0.0
+0-60s,34029,34029,0,0.0,0,0.0,0.0
+60-120s,239636,21149,321,0.13,218166,91.04,1.5
+120-180s,137004,16881,334,0.24,119789,87.43,2.0
+180-240s,138916,15348,366,0.26,123202,88.69,2.5
+240-300s,139550,12993,404,0.29,126153,90.4,3.0
+300-360s,138927,12114,404,0.29,126409,90.99,3.5
+360-420s,138982,10570,454,0.33,127958,92.07,4.0
+420-480s,134916,9866,433,0.32,124617,92.37,4.5
+480-540s,99929,9211,459,0.46,90259,90.32,5.0
+540-600s,54660,29856,6,0.01,24798,45.37,0.0