Pass --timeout flag to pyperf (#354)

* Implement timeout mechanism for a benchmark run

* Address feedbacks

* Pin pyperf to 2.8.0

Update test_commands to reflect a change how pyperf reports data.

doc/usage.rst

@@ -102,8 +102,8 @@ Usage::
   pyperformance run [-h] [-r] [-f] [--debug-single-value] [-v] [-m]
                        [--affinity CPU_LIST] [-o FILENAME]
                        [--append FILENAME] [--manifest MANIFEST]
-                       [-b BM_LIST] [--inherit-environ VAR_LIST]
-                       [-p PYTHON]
+                       [--timeout TIMEOUT] [-b BM_LIST]
+                       [--inherit-environ VAR_LIST] [-p PYTHON]
 
 options::
 
@@ -124,6 +124,8 @@ options::
                         baseline_python, not changed_python.
   --append FILENAME     Add runs to an existing file, or create it if
                         it doesn't exist
+  --timeout TIMEOUT     Specify a timeout in seconds for a single
+                        benchmark run (default: disabled)
   --manifest MANIFEST   benchmark manifest file to use
   -b BM_LIST, --benchmarks BM_LIST
                         Comma-separated list of benchmarks to run. Can

pyperformance/_benchmark.py

@@ -233,7 +233,11 @@ def _run_perf_script(python, runscript, runid, *,
                 sys.stderr.flush()
                 sys.stderr.write(stderr)
                 sys.stderr.flush()
-            raise RuntimeError("Benchmark died")
+            # pyperf returns exit code 124 if the benchmark execution times out
+            if ec == 124:
+                raise TimeoutError("Benchmark timed out")
+            else:
+                raise RuntimeError("Benchmark died")
         return pyperf.BenchmarkSuite.load(tmp)
 
 

pyperformance/cli.py

@@ -25,6 +25,13 @@ def comma_separated(values):
     return list(filter(None, values))
 
 
+def check_positive(value):
+    value = int(value)
+    if value <= 0:
+        raise argparse.ArgumentTypeError("Argument must a be positive integer.")
+    return value
+
+
 def filter_opts(cmd, *, allow_no_benchmarks=False):
     cmd.add_argument("--manifest", help="benchmark manifest file to use")
 
@@ -82,6 +89,10 @@ def parse_args():
                      help="Use the same number of loops as a previous run "
                      "(i.e., don't recalibrate). Should be a path to a "
                      ".json file from a previous run.")
+    cmd.add_argument("--timeout",
+                     help="Specify a timeout in seconds for a single "
+                     "benchmark run (default: disabled)",
+                     type=check_positive)
     filter_opts(cmd)
 
     # show

pyperformance/commands.py

@@ -191,8 +191,8 @@ def cmd_run(options, benchmarks):
 
     if errors:
         print("%s benchmarks failed:" % len(errors))
-        for name in errors:
-            print("- %s" % name)
+        for name, reason in errors:
+            print("- %s (%s)" % (name, reason))
         print()
         sys.exit(1)
 

pyperformance/requirements/requirements.txt

@@ -10,5 +10,5 @@ psutil==5.9.5
     # via
     #   -r requirements.in
     #   pyperf
-pyperf==2.7.0
+pyperf==2.8.0
     # via -r requirements.in

pyperformance/run.py

@@ -164,7 +164,7 @@ def add_bench(dest_suite, obj):
         bench_venv, bench_runid = benchmarks.get(bench)
         if bench_venv is None:
             print("ERROR: Benchmark %s failed: could not install requirements" % name)
-            errors.append(name)
+            errors.append((name, "Install requirements error"))
             continue
         try:
             result = bench.run(
@@ -174,10 +174,17 @@ def add_bench(dest_suite, obj):
                 venv=bench_venv,
                 verbose=options.verbose,
             )
+        except TimeoutError as exc:
+            print("ERROR: Benchmark %s timed out" % name)
+            errors.append((name, exc))
+        except RuntimeError as exc:
+            print("ERROR: Benchmark %s failed: %s" % (name, exc))
+            traceback.print_exc()
+            errors.append((name, exc))
         except Exception as exc:
             print("ERROR: Benchmark %s failed: %s" % (name, exc))
             traceback.print_exc()
-            errors.append(name)
+            errors.append((name, exc))
         else:
             suite = add_bench(suite, result)
 
@@ -233,5 +240,7 @@ def get_pyperf_opts(options):
         opts.append('--inherit-environ=%s' % ','.join(options.inherit_environ))
     if options.min_time:
         opts.append('--min-time=%s' % options.min_time)
+    if options.timeout:
+        opts.append('--timeout=%s' % options.timeout)
 
     return opts

pyperformance/tests/test_commands.py

@@ -399,7 +399,7 @@ def test_compare_single_value(self):
             Performance version: 0.2
 
             ### call_simple ###
-            7896.0 kB -> 7900.0 kB: 1.00x larger
+            7896.0 KiB -> 7900.0 KiB: 1.00x larger
         ''').lstrip())
 
     def test_compare_csv(self):
@@ -458,11 +458,11 @@ def test_compare_table_single_value(self):
 
             Performance version: 0.2
 
-            +-------------+-----------+-----------+--------------+------------------------------------------+
-            | Benchmark   | mem1.json | mem2.json | Change       | Significance                             |
-            +=============+===========+===========+==============+==========================================+
-            | call_simple | 7896.0 kB | 7900.0 kB | 1.00x larger | (benchmark only contains a single value) |
-            +-------------+-----------+-----------+--------------+------------------------------------------+
+            +-------------+------------+------------+--------------+------------------------------------------+
+            | Benchmark   | mem1.json  | mem2.json  | Change       | Significance                             |
+            +=============+============+============+==============+==========================================+
+            | call_simple | 7896.0 KiB | 7900.0 KiB | 1.00x larger | (benchmark only contains a single value) |
+            +-------------+------------+------------+--------------+------------------------------------------+
         ''').lstrip())