Add worst known feasible value to constrained problems

botorch/test_functions/base.py

@@ -239,6 +239,7 @@ class ConstrainedBaseTestProblem(BaseTestProblem, ABC):
     num_constraints: int
     _check_grad_at_opt: bool = False
     constraint_noise_std: None | float | list[float] = None
+    _worst_feasible_value: float | None = None
 
     def evaluate_slack(self, X: Tensor, noise: bool = True) -> Tensor:
         r"""Evaluate the constraint slack on a set of points.
@@ -313,6 +314,26 @@ def _evaluate_slack_true(self, X: Tensor) -> Tensor:
         """
         pass  # pragma: no cover
 
+    @property
+    def worst_feasible_value(self) -> float:
+        r"""The worst feasible value of the objective function. This is useful when
+        evaluating the performance of different optimization methods as this value
+        can be assigned to all infeasible trials. This has the desirable property that
+        any feasible trial has better performance than an infeasible trial.
+        """
+        if isinstance(self, MultiObjectiveTestProblem):
+            return 0.0  # Can return 0.0 for MOO since this is the smallest possible HV
+        elif self._worst_feasible_value is not None:
+            return (
+                -self._worst_feasible_value
+                if self.negate
+                else self._worst_feasible_value
+            )
+        raise NotImplementedError(
+            f"Problem {self.__class__.__name__} does not specify the "
+            "worst feasible value."
+        )
+
 
 class MultiObjectiveTestProblem(BaseTestProblem, ABC):
     r"""Base class for multi-objective test functions.

botorch/test_functions/synthetic.py

@@ -1064,6 +1064,7 @@ class ConstrainedGramacy(ConstrainedSyntheticTestFunction):
     _bounds = [(0.0, 1.0), (0.0, 1.0)]
     _optimizers = [(0.1954, 0.4044)]
     _optimal_value = 0.5998  # approximate from [Gramacy2016]_
+    _worst_feasible_value = 1.732051  # Computed from 100 SLSQP restarts
 
     def _evaluate_true(self, X: Tensor) -> Tensor:
         """
@@ -1122,6 +1123,10 @@ def __init__(
         self.constraint_noise_std = self._validate_constraint_noise(
             constraint_noise_std
         )
+        if dim == 3:
+            self._worst_feasible_value = -0.0002735  # Computed from 100 SLSQP restarts
+        elif dim == 6:
+            self._worst_feasible_value = -0.0001346  # Computed from 100 SLSQP restarts
 
     def _evaluate_slack_true(self, X: Tensor) -> Tensor:
         return -X.norm(dim=-1, keepdim=True) + 1
@@ -1167,6 +1172,10 @@ def __init__(
         self.constraint_noise_std = self._validate_constraint_noise(
             constraint_noise_std
         )
+        if dim == 3:
+            self._worst_feasible_value = -0.0002735  # Computed from 100 SLSQP restarts
+        elif dim == 6:
+            self._worst_feasible_value = -0.0001346  # Computed from 100 SLSQP restarts
 
     def _evaluate_slack_true(self, X: Tensor) -> Tensor:
         return -X.pow(2).sum(dim=-1, keepdim=True) + 1
@@ -1184,6 +1193,7 @@ class PressureVessel(ConstrainedSyntheticTestFunction):
     num_constraints = 4
     _bounds = [(0.0, 10.0), (0.0, 10.0), (10.0, 50.0), (150.0, 200.0)]
     _optimal_value = 6059.946341  # from [CoelloCoello2002constraint]
+    _worst_feasible_value = 240526.7248  # Computed from 100 SLSQP restarts
 
     def _evaluate_true(self, X: Tensor) -> Tensor:
         x1, x2, x3, x4 = X.unbind(-1)
@@ -1224,6 +1234,7 @@ class WeldedBeamSO(ConstrainedSyntheticTestFunction):
     num_constraints = 6
     _bounds = [(0.125, 10.0), (0.1, 10.0), (0.1, 10.0), (0.1, 10.0)]
     _optimal_value = 1.728226  # from [CoelloCoello2002constraint]
+    _worst_feasible_value = 19.01859  # Computed from 100 SLSQP restarts
 
     def _evaluate_true(self, X: Tensor) -> Tensor:
         x1, x2, x3, x4 = X.unbind(-1)
@@ -1279,6 +1290,7 @@ class TensionCompressionString(ConstrainedSyntheticTestFunction):
     num_constraints = 4
     _bounds = [(0.01, 1.0), (0.01, 1.0), (0.01, 20.0)]
     _optimal_value = 0.012681  # from [CoelloCoello2002constraint]
+    _worst_feasible_value = 0.306081  # Computed from 100 SLSQP restarts
 
     def _evaluate_true(self, X: Tensor) -> Tensor:
         x1, x2, x3 = X.unbind(-1)
@@ -1320,6 +1332,7 @@ class SpeedReducer(ConstrainedSyntheticTestFunction):
         (5.0, 5.5),
     ]
     _optimal_value = 2996.3482  # from [Lemonge2010constrained]
+    _worst_feasible_value = 6117.3420  # Computed from 100 SLSQP restarts
 
     def _evaluate_true(self, X: Tensor) -> Tensor:
         x1, x2, x3, x4, x5, x6, x7 = X.unbind(-1)
@@ -1377,6 +1390,7 @@ class KeaneBumpFunction(ConstrainedSyntheticTestFunction):
         40: -0.826624404,
         50: -0.83078783,
     }
+    _worst_feasible_value = 0.0  # Computed from 100 SLSQP restarts
 
     def __init__(
         self,

botorch/utils/testing.py

@@ -33,6 +33,7 @@
     BaseTestProblem,
     ConstrainedBaseTestProblem,
     CorruptedTestProblem,
+    MultiObjectiveTestProblem,
 )
 from botorch.test_functions.synthetic import Rosenbrock
 from botorch.utils.transforms import unnormalize
@@ -402,6 +403,25 @@ def test_evaluate_slack(self):
                 else:
                     self.assertTrue(is_equal.all().item())
 
+    def test_worst_feasible_value(self):
+        """Test that a function's worst_feasible_value is correctly computed,
+        and defined if it should be.
+        """
+        for dtype in (torch.float, torch.double):
+            for f in self.functions:
+                f.to(device=self.device, dtype=dtype)
+                if f._worst_feasible_value is None:
+                    self.assertTrue(isinstance(f, MultiObjectiveTestProblem))
+                    self.assertGreaterEqual(f.worst_feasible_value, 0.0)
+                else:
+                    worst_feas_val = f.worst_feasible_value
+                    worst_feas_val_exp = (
+                        -f._worst_feasible_value
+                        if f.negate
+                        else f._worst_feasible_value
+                    )
+                    self.assertEqual(worst_feas_val, worst_feas_val_exp)
+
     @property
     @abstractmethod
     def functions(self) -> Sequence[BaseTestProblem]:

test/test_functions/test_synthetic.py

@@ -144,6 +144,10 @@ def test_constraint_noise_length_validation(self):
         ):
             DummyConstrainedSyntheticTestFunction(constraint_noise_std=[0.1, 0.2])
 
+    def test_worst_feasible_value_not_implemented(self):
+        with self.assertRaises(NotImplementedError):
+            DummyConstrainedSyntheticTestFunction().worst_feasible_value
+
 
 class TestAckley(
     BotorchTestCase, BaseTestProblemTestCaseMixIn, SyntheticTestFunctionTestCaseMixin
@@ -387,13 +391,14 @@ class TestConstrainedHartmann(
     SyntheticTestFunctionTestCaseMixin,
     ConstrainedTestProblemTestCaseMixin,
 ):
-    functions = [
-        ConstrainedHartmann(dim=6, negate=True),
-        ConstrainedHartmann(noise_std=0.1, dim=6, negate=True),
-        ConstrainedHartmann(
-            noise_std=0.1, constraint_noise_std=0.2, dim=6, negate=True
-        ),
-    ]
+    for dim in [3, 6]:
+        functions = [
+            ConstrainedHartmann(dim=dim, negate=True),
+            ConstrainedHartmann(noise_std=0.1, dim=dim, negate=True),
+            ConstrainedHartmann(
+                noise_std=0.1, constraint_noise_std=0.2, dim=dim, negate=True
+            ),
+        ]
 
 
 class TestConstrainedHartmannSmooth(
@@ -402,12 +407,13 @@ class TestConstrainedHartmannSmooth(
     SyntheticTestFunctionTestCaseMixin,
     ConstrainedTestProblemTestCaseMixin,
 ):
-    functions = [
-        ConstrainedHartmannSmooth(dim=6, negate=True),
-        ConstrainedHartmannSmooth(
-            dim=6, noise_std=0.1, constraint_noise_std=0.2, negate=True
-        ),
-    ]
+    for dim in [3, 6]:
+        functions = [
+            ConstrainedHartmannSmooth(dim=dim, negate=True),
+            ConstrainedHartmannSmooth(
+                dim=dim, noise_std=0.1, constraint_noise_std=0.2, negate=True
+            ),
+        ]
 
 
 class TestPressureVessel(