Add multiobjective experimenter factory and extend capabilities for state analyzers.

PiperOrigin-RevId: 647087713

Author: qiuyiz

vizier/_src/benchmarks/analyzers/convergence_curve.py

@@ -355,8 +355,9 @@ def __init__(
     Args:
       metric_informations:
       reference_value: Reference point value from which hypervolume is computed,
-        with shape that is broadcastable with (dim,). If None, this computes the
-        minimum of each objective as the reference point.
+        with shape that is broadcastable with (dim,). Note that the sign is
+        flipped for minimization metrics. If None, this computes the minimum of
+        each objective as the reference point.
       num_vectors: Number of vectors from which hypervolume is computed.
       infer_origin_factor: When inferring the reference point, set origin to be
         minimum value - factor * (range).

vizier/_src/benchmarks/analyzers/state_analyzer.py

@@ -92,6 +92,7 @@ def to_curve(
       cls,
       states: list[benchmarks.BenchmarkState],
       flip_signs_for_min: bool = False,
+      reference_value: Optional[np.ndarray] = None,
   ) -> convergence_curve.ConvergenceCurve:
     """Generates a ConvergenceCurve from a batch of BenchmarkStates.
 
@@ -101,6 +102,7 @@ def to_curve(
       states: List of BenchmarkStates.
       flip_signs_for_min: If true, flip signs of curve when it is MINIMIZE
         metric.
+      reference_value: Reference value for multiobjective hypervolume curve.
 
     Returns:
       Convergence curve with batch size equal to length of states.
@@ -122,10 +124,15 @@ def to_curve(
         )
       state_trials = state.algorithm.supporter.GetTrials()
 
+      if problem_statement.is_single_objective:
+        kwargs = {'flip_signs_for_min': flip_signs_for_min}
+      else:
+        kwargs = {'reference_value': reference_value}
+
       converter = (
           convergence_curve.MultiMetricCurveConverter.from_metrics_config(
               problem_statement.metric_information,
-              flip_signs_for_min=flip_signs_for_min,
+              **kwargs,
           )
       )
       curve = converter.convert(state_trials)

vizier/_src/benchmarks/analyzers/state_analyzer_test.py

@@ -16,6 +16,7 @@
 
 import itertools
 import json
+import numpy as np
 from vizier import benchmarks as vzb
 from vizier import pyvizier as vz
 from vizier._src.algorithms.designers import grid
@@ -57,6 +58,40 @@ def test_empty_curve_error(self):
     with self.assertRaisesRegex(ValueError, 'Empty'):
       state_analyzer.BenchmarkStateAnalyzer.to_curve([])
 
+  def test_multiobj_curve_conversion(self):
+    dim = 10
+    experimenter_factories = {
+        'sphere': experimenters.BBOBExperimenterFactory('Sphere', dim),
+        'discus': experimenters.BBOBExperimenterFactory('Discus', dim),
+    }
+    multi_experimenter = experimenters.MultiObjectiveExperimenterFactory(
+        base_factories=experimenter_factories
+    )()
+
+    def _designer_factory(config: vz.ProblemStatement, seed: int):
+      return random.RandomDesigner(config.search_space, seed=seed)
+
+    benchmark_state_factory = vzb.DesignerBenchmarkStateFactory(
+        designer_factory=_designer_factory, experimenter=multi_experimenter
+    )
+    num_trials = 20
+    runner = vzb.BenchmarkRunner(
+        benchmark_subroutines=[vzb.GenerateAndEvaluate()],
+        num_repeats=num_trials,
+    )
+
+    states = []
+    num_repeats = 3
+    for i in range(num_repeats):
+      bench_state = benchmark_state_factory(seed=i)
+      runner.run(bench_state)
+      states.append(bench_state)
+
+    curve = state_analyzer.BenchmarkStateAnalyzer.to_curve(
+        states, reference_value=np.asarray([-1])
+    )
+    self.assertEqual(curve.ys.shape, (num_repeats, num_trials))
+
   def test_different_curve_error(self):
     exp1 = experimenters.BBOBExperimenterFactory('Sphere', dim=2)()
     exp2 = experimenters.BBOBExperimenterFactory('Sphere', dim=3)()

vizier/_src/benchmarks/experimenters/experimenter_factory.py

@@ -26,6 +26,7 @@
 from vizier import pyvizier as vz
 from vizier._src.benchmarks.experimenters import discretizing_experimenter
 from vizier._src.benchmarks.experimenters import experimenter
+from vizier._src.benchmarks.experimenters import multiobjective_experimenter
 from vizier._src.benchmarks.experimenters import noisy_experimenter
 from vizier._src.benchmarks.experimenters import normalizing_experimenter
 from vizier._src.benchmarks.experimenters import numpy_experimenter
@@ -37,6 +38,7 @@
 
 BBOB_FACTORY_KEY = 'bbob_factory'
 SINGLE_OBJECTIVE_FACTORY_KEY = 'single_objective_factory'
+MULTI_OBJECTIVE_FACTORY_KEY = 'multi_objective_factory'
 
 
 class ExperimenterFactory(abc.ABC):
@@ -248,3 +250,44 @@ def recover(
     return SingleObjectiveExperimenterFactory(
         base_factory=base_factory, **metadata_dict
     )
+
+
+@attr.define
+class MultiObjectiveExperimenterFactory(SerializableExperimenterFactory):
+  """Factory for a multi-objective Experimenter that combines multiple single-objective experimenters.
+
+  Attributes:
+    base_factories:
+  """
+
+  base_factories: dict[str, SerializableExperimenterFactory] = attr.field()
+
+  def __call__(self) -> experimenter.Experimenter:
+    """Creates the MultiObjective Experimenter."""
+    exptrs = {name: factory() for name, factory in self.base_factories.items()}
+    return multiobjective_experimenter.MultiObjectiveExperimenter(exptrs)
+
+  def dump(self) -> vz.Metadata:
+    metadata = vz.Metadata()
+    metadata_dict = {
+        name: factory.dump() for name, factory in self.base_factories.items()
+    }
+    metadata[MULTI_OBJECTIVE_FACTORY_KEY] = json.dumps(
+        metadata_dict, cls=json_utils.NumpyEncoder
+    )
+    return metadata
+
+  @classmethod
+  def recover(
+      cls, metadata: vz.Metadata
+  ) -> 'MultiObjectiveExperimenterFactory':
+    # TODO: Use generics to make this work.
+    metadata_dict = json.loads(
+        metadata[MULTI_OBJECTIVE_FACTORY_KEY], cls=json_utils.NumpyDecoder
+    )
+    return MultiObjectiveExperimenterFactory(
+        base_factories={
+            name: SerializableExperimenterFactory.recover(factory_dump)
+            for name, factory_dump in metadata_dict.items()
+        }
+    )

vizier/_src/benchmarks/experimenters/experimenter_factory_test.py

@@ -74,6 +74,29 @@ def testSingleObjectiveFactory(self):
     exptr.evaluate([t])
     self.assertEqual(t.status, pyvizier.TrialStatus.COMPLETED)
 
+  def testMultiObjectiveFactory(self):
+    dim = 5
+    experimenter_factories = {
+        'sphere': experimenter_factory.BBOBExperimenterFactory('Sphere', dim),
+        'discus': experimenter_factory.BBOBExperimenterFactory('Discus', dim),
+    }
+    exptr = experimenter_factory.MultiObjectiveExperimenterFactory(
+        base_factories=experimenter_factories
+    )()
+
+    parameters = exptr.problem_statement().search_space.parameters
+    self.assertLen(parameters, dim)
+
+    t = pyvizier.Trial(
+        parameters={
+            param.name: float(index) for index, param in enumerate(parameters)
+        }
+    )
+    exptr.evaluate([t])
+    self.assertIn('sphere', t.final_measurement_or_die.metrics)
+    self.assertIn('discus', t.final_measurement_or_die.metrics)
+    self.assertEqual(t.status, pyvizier.TrialStatus.COMPLETED)
+
   def testSingleObjectiveFactoryDiscrete(self):
     dim = 5
     bbob_factory = experimenter_factory.BBOBExperimenterFactory(

vizier/_src/benchmarks/experimenters/multiobjective_experimenter.py

@@ -61,10 +61,10 @@ def __init__(
 
     metric_infos = []
     # Keeps track of the underlying metric information name of each extpr.
-    self._previous_names = {}
+    self._exptr_to_metric = {}
     for name, exptr in exptrs.items():
       metric_info = exptr.problem_statement().metric_information.item()
-      self._previous_names[name] = metric_info.name
+      self._exptr_to_metric[name] = metric_info.name
       metric_info.name = name
       metric_infos.append(metric_info)
 
@@ -80,12 +80,12 @@ def evaluate(self, suggestions: Sequence[pyvizier.Trial]):
     measurements = [pyvizier.Measurement() for _ in suggestions]
     for name, exptr in self._exptrs.items():
       exptr.evaluate(suggestions_copy)
-      previous_name = self._previous_names[name]
+      exptr_metric_name = self._exptr_to_metric[name]
       for idx, copied in enumerate(suggestions_copy):
         measurement = measurements[idx]
         assert copied.final_measurement is not None
         measurement.metrics[name] = copied.final_measurement.metrics[
-            previous_name
+            exptr_metric_name
         ]
 
     for suggestion, measurement in zip(suggestions, measurements):

vizier/benchmarks/experimenters/__init__.py

@@ -26,6 +26,7 @@
 from vizier._src.benchmarks.experimenters.experimenter import Experimenter
 from vizier._src.benchmarks.experimenters.experimenter_factory import BBOBExperimenterFactory
 from vizier._src.benchmarks.experimenters.experimenter_factory import ExperimenterFactory
+from vizier._src.benchmarks.experimenters.experimenter_factory import MultiObjectiveExperimenterFactory
 from vizier._src.benchmarks.experimenters.experimenter_factory import SerializableExperimenterFactory
 from vizier._src.benchmarks.experimenters.experimenter_factory import SingleObjectiveExperimenterFactory
 from vizier._src.benchmarks.experimenters.infeasible_experimenter import HashingInfeasibleExperimenter