Move max_size into config

Author: staadecker

benchmarks/Snakefile

@@ -16,7 +16,7 @@ def generate_all_runs(problem):
 
     problem_data = config["problems"][problem]
     if problem_data is None or "size" not in problem_data:
-        sizes = ["0"]
+        sizes = [0]
     else:
         sizes = problem_data["size"]
 
@@ -27,19 +27,7 @@ def generate_all_runs(problem):
             config["libraries"],
             config["solvers"]
         )
-    ]
-
-    benchmarks = [
-            importlib.import_module(f"benchmarks.{problem}.bm_{library.lower()}").Bench 
-            if library not in ["jump"]
-            else None
-            for size, library, solver in runs
-    ]
-
-    runs = [
-        r for r, benchmark in zip(
-            runs, benchmarks
-        ) if benchmark is None or benchmark.MAX_SIZE == None or r[0] <= benchmark.MAX_SIZE
+        if size <= problem_data["max_sizes"].get(library, float('inf'))
     ]
 
     return runs
@@ -84,7 +72,7 @@ rule mem_profile:
         "results/{problem}/{library}_{solver}_{size}_memray.html"
     shell:
         """
-        memray run --trace-python-allocators --native --aggregate -o results/{wildcards.problem}/{wildcards.library}_{wildcards.solver}_{wildcards.size}_memray.bin src/run_benchmark.py {wildcards.problem} --library {wildcards.library} --solver {wildcards.solver} --size {wildcards.size}
+        memray run --trace-python-allocators --follow-fork --native --aggregate -o results/{wildcards.problem}/{wildcards.library}_{wildcards.solver}_{wildcards.size}_memray.bin src/run_benchmark.py {wildcards.problem} --library {wildcards.library} --solver {wildcards.solver} --size {wildcards.size}
         memray flamegraph -o results/{wildcards.problem}/{wildcards.library}_{wildcards.solver}_{wildcards.size}_memray.html results/{wildcards.problem}/{wildcards.library}_{wildcards.solver}_{wildcards.size}_memray.bin
         """
 

benchmarks/config.yaml

@@ -1,4 +1,4 @@
-repeat: 1  # TODO repeat
+repeat: 1 # TODO repeat
 problems:
   facility_location:
     size:
@@ -7,6 +7,11 @@ problems:
       - 75
       - 100
       - 150
+      - 200
+    max_sizes:
+      # jump: 150
+      pyomo: 75
+      gurobipy: 150
   # energy_problem:
 
 solvers:
@@ -27,4 +32,4 @@ profile:
   - library: pyoframe
     solver: gurobi
     size: 150
-    problem: facility_location
+    problem: facility_location

benchmarks/src/facility_location/bm_gurobipy.py

@@ -9,8 +9,6 @@
 
 
 class Bench(GurobiPyBenchmark):
-    MAX_SIZE = 100
-
     def build(self):
         if isinstance(self.size, int):
             G = F = self.size

benchmarks/src/facility_location/bm_pyomo.py

@@ -10,8 +10,6 @@
 
 
 class Bench(PyomoBenchmark):
-    MAX_SIZE = 75  # After that it gets too slow
-
     def build(self):
         if isinstance(self.size, int):
             G = F = self.size

benchmarks/src/run_benchmark.py

@@ -32,15 +32,6 @@ def get_problems() -> Collection[str]:
             f"benchmarks.{args.problem}.bm_{args.library.lower()}"
         ).Bench
 
-        if (
-            args.size is not None
-            and benchmark.MAX_SIZE is not None
-            and args.size > benchmark.MAX_SIZE
-        ):
-            raise ValueError(
-                f"Size {args.size} exceeds maximum size {benchmark.MAX_SIZE} for problem {args.problem}."
-            )
-
         benchmark(solver=args.solver, size=args.size).run()
     else:
         subprocess.run(

benchmarks/src/utils.py

@@ -8,8 +8,6 @@
 
 
 class Benchmark(ABC):
-    MAX_SIZE = None
-
     def __init__(self, solver, size=None, block_solver=True):
         assert solver in self.get_supported_solvers(), (
             f"{solver} is not supported by {self.__class__.__name__}."