From f75a4fdd713b05af110723a8339651e1d6d3879d Mon Sep 17 00:00:00 2001 From: Richard Zhang Date: Wed, 26 Jun 2024 14:55:10 -0700 Subject: [PATCH] Add multiobjective experimenter factory and extend capabilities for state analyzers. PiperOrigin-RevId: 647087713 --- .../benchmarks/analyzers/convergence_curve.py | 5 ++- .../benchmarks/analyzers/state_analyzer.py | 9 +++- .../analyzers/state_analyzer_test.py | 35 ++++++++++++++++ .../experimenters/experimenter_factory.py | 41 +++++++++++++++++++ .../experimenter_factory_test.py | 23 +++++++++++ .../multiobjective_experimenter.py | 8 ++-- vizier/benchmarks/experimenters/__init__.py | 1 + 7 files changed, 115 insertions(+), 7 deletions(-) diff --git a/vizier/_src/benchmarks/analyzers/convergence_curve.py b/vizier/_src/benchmarks/analyzers/convergence_curve.py index 4720e6d06..907a0cfa4 100644 --- a/vizier/_src/benchmarks/analyzers/convergence_curve.py +++ b/vizier/_src/benchmarks/analyzers/convergence_curve.py @@ -355,8 +355,9 @@ def __init__( Args: metric_informations: reference_value: Reference point value from which hypervolume is computed, - with shape that is broadcastable with (dim,). If None, this computes the - minimum of each objective as the reference point. + with shape that is broadcastable with (dim,). Note that the sign is + flipped for minimization metrics. If None, this computes the minimum of + each objective as the reference point. num_vectors: Number of vectors from which hypervolume is computed. infer_origin_factor: When inferring the reference point, set origin to be minimum value - factor * (range). diff --git a/vizier/_src/benchmarks/analyzers/state_analyzer.py b/vizier/_src/benchmarks/analyzers/state_analyzer.py index 1d6d321b7..d6305c8c6 100644 --- a/vizier/_src/benchmarks/analyzers/state_analyzer.py +++ b/vizier/_src/benchmarks/analyzers/state_analyzer.py @@ -92,6 +92,7 @@ def to_curve( cls, states: list[benchmarks.BenchmarkState], flip_signs_for_min: bool = False, + reference_value: Optional[np.ndarray] = None, ) -> convergence_curve.ConvergenceCurve: """Generates a ConvergenceCurve from a batch of BenchmarkStates. @@ -101,6 +102,7 @@ def to_curve( states: List of BenchmarkStates. flip_signs_for_min: If true, flip signs of curve when it is MINIMIZE metric. + reference_value: Reference value for multiobjective hypervolume curve. Returns: Convergence curve with batch size equal to length of states. @@ -122,10 +124,15 @@ def to_curve( ) state_trials = state.algorithm.supporter.GetTrials() + if problem_statement.is_single_objective: + kwargs = {'flip_signs_for_min': flip_signs_for_min} + else: + kwargs = {'reference_value': reference_value} + converter = ( convergence_curve.MultiMetricCurveConverter.from_metrics_config( problem_statement.metric_information, - flip_signs_for_min=flip_signs_for_min, + **kwargs, ) ) curve = converter.convert(state_trials) diff --git a/vizier/_src/benchmarks/analyzers/state_analyzer_test.py b/vizier/_src/benchmarks/analyzers/state_analyzer_test.py index 6968e9e0e..753539cbb 100644 --- a/vizier/_src/benchmarks/analyzers/state_analyzer_test.py +++ b/vizier/_src/benchmarks/analyzers/state_analyzer_test.py @@ -16,6 +16,7 @@ import itertools import json +import numpy as np from vizier import benchmarks as vzb from vizier import pyvizier as vz from vizier._src.algorithms.designers import grid @@ -57,6 +58,40 @@ def test_empty_curve_error(self): with self.assertRaisesRegex(ValueError, 'Empty'): state_analyzer.BenchmarkStateAnalyzer.to_curve([]) + def test_multiobj_curve_conversion(self): + dim = 10 + experimenter_factories = { + 'sphere': experimenters.BBOBExperimenterFactory('Sphere', dim), + 'discus': experimenters.BBOBExperimenterFactory('Discus', dim), + } + multi_experimenter = experimenters.CombinedExperimenterFactory( + base_factories=experimenter_factories + )() + + def _designer_factory(config: vz.ProblemStatement, seed: int): + return random.RandomDesigner(config.search_space, seed=seed) + + benchmark_state_factory = vzb.DesignerBenchmarkStateFactory( + designer_factory=_designer_factory, experimenter=multi_experimenter + ) + num_trials = 20 + runner = vzb.BenchmarkRunner( + benchmark_subroutines=[vzb.GenerateAndEvaluate()], + num_repeats=num_trials, + ) + + states = [] + num_repeats = 3 + for i in range(num_repeats): + bench_state = benchmark_state_factory(seed=i) + runner.run(bench_state) + states.append(bench_state) + + curve = state_analyzer.BenchmarkStateAnalyzer.to_curve( + states, reference_value=np.asarray([-1]) + ) + self.assertEqual(curve.ys.shape, (num_repeats, num_trials)) + def test_different_curve_error(self): exp1 = experimenters.BBOBExperimenterFactory('Sphere', dim=2)() exp2 = experimenters.BBOBExperimenterFactory('Sphere', dim=3)() diff --git a/vizier/_src/benchmarks/experimenters/experimenter_factory.py b/vizier/_src/benchmarks/experimenters/experimenter_factory.py index 9d92bccfc..b33e91304 100644 --- a/vizier/_src/benchmarks/experimenters/experimenter_factory.py +++ b/vizier/_src/benchmarks/experimenters/experimenter_factory.py @@ -26,6 +26,7 @@ from vizier import pyvizier as vz from vizier._src.benchmarks.experimenters import discretizing_experimenter from vizier._src.benchmarks.experimenters import experimenter +from vizier._src.benchmarks.experimenters import multiobjective_experimenter from vizier._src.benchmarks.experimenters import noisy_experimenter from vizier._src.benchmarks.experimenters import normalizing_experimenter from vizier._src.benchmarks.experimenters import numpy_experimenter @@ -37,6 +38,7 @@ BBOB_FACTORY_KEY = 'bbob_factory' SINGLE_OBJECTIVE_FACTORY_KEY = 'single_objective_factory' +MULTI_OBJECTIVE_FACTORY_KEY = 'multi_objective_factory' class ExperimenterFactory(abc.ABC): @@ -248,3 +250,42 @@ def recover( return SingleObjectiveExperimenterFactory( base_factory=base_factory, **metadata_dict ) + + +@attr.define +class CombinedExperimenterFactory(SerializableExperimenterFactory): + """Factory for a multi-objective Experimenter that combines multiple single-objective experimenters. + + Attributes: + base_factories: + """ + + base_factories: dict[str, SerializableExperimenterFactory] = attr.field() + + def __call__(self) -> experimenter.Experimenter: + """Creates the MultiObjective Experimenter.""" + exptrs = {name: factory() for name, factory in self.base_factories.items()} + return multiobjective_experimenter.MultiObjectiveExperimenter(exptrs) + + def dump(self) -> vz.Metadata: + metadata = vz.Metadata() + metadata_dict = { + name: factory.dump() for name, factory in self.base_factories.items() + } + metadata[MULTI_OBJECTIVE_FACTORY_KEY] = json.dumps( + metadata_dict, cls=json_utils.NumpyEncoder + ) + return metadata + + @classmethod + def recover(cls, metadata: vz.Metadata) -> 'CombinedExperimenterFactory': + # TODO: Use generics to make this work. + metadata_dict = json.loads( + metadata[MULTI_OBJECTIVE_FACTORY_KEY], cls=json_utils.NumpyDecoder + ) + return CombinedExperimenterFactory( + base_factories={ + name: SerializableExperimenterFactory.recover(factory_dump) + for name, factory_dump in metadata_dict.items() + } + ) diff --git a/vizier/_src/benchmarks/experimenters/experimenter_factory_test.py b/vizier/_src/benchmarks/experimenters/experimenter_factory_test.py index 69ec214eb..c533b9c3f 100644 --- a/vizier/_src/benchmarks/experimenters/experimenter_factory_test.py +++ b/vizier/_src/benchmarks/experimenters/experimenter_factory_test.py @@ -74,6 +74,29 @@ def testSingleObjectiveFactory(self): exptr.evaluate([t]) self.assertEqual(t.status, pyvizier.TrialStatus.COMPLETED) + def testCombinedFactory(self): + dim = 5 + experimenter_factories = { + 'sphere': experimenter_factory.BBOBExperimenterFactory('Sphere', dim), + 'discus': experimenter_factory.BBOBExperimenterFactory('Discus', dim), + } + exptr = experimenter_factory.CombinedExperimenterFactory( + base_factories=experimenter_factories + )() + + parameters = exptr.problem_statement().search_space.parameters + self.assertLen(parameters, dim) + + t = pyvizier.Trial( + parameters={ + param.name: float(index) for index, param in enumerate(parameters) + } + ) + exptr.evaluate([t]) + self.assertIn('sphere', t.final_measurement_or_die.metrics) + self.assertIn('discus', t.final_measurement_or_die.metrics) + self.assertEqual(t.status, pyvizier.TrialStatus.COMPLETED) + def testSingleObjectiveFactoryDiscrete(self): dim = 5 bbob_factory = experimenter_factory.BBOBExperimenterFactory( diff --git a/vizier/_src/benchmarks/experimenters/multiobjective_experimenter.py b/vizier/_src/benchmarks/experimenters/multiobjective_experimenter.py index 8a1a08c9a..6b8696f5c 100644 --- a/vizier/_src/benchmarks/experimenters/multiobjective_experimenter.py +++ b/vizier/_src/benchmarks/experimenters/multiobjective_experimenter.py @@ -61,10 +61,10 @@ def __init__( metric_infos = [] # Keeps track of the underlying metric information name of each extpr. - self._previous_names = {} + self._exptr_to_metric = {} for name, exptr in exptrs.items(): metric_info = exptr.problem_statement().metric_information.item() - self._previous_names[name] = metric_info.name + self._exptr_to_metric[name] = metric_info.name metric_info.name = name metric_infos.append(metric_info) @@ -80,12 +80,12 @@ def evaluate(self, suggestions: Sequence[pyvizier.Trial]): measurements = [pyvizier.Measurement() for _ in suggestions] for name, exptr in self._exptrs.items(): exptr.evaluate(suggestions_copy) - previous_name = self._previous_names[name] + exptr_metric_name = self._exptr_to_metric[name] for idx, copied in enumerate(suggestions_copy): measurement = measurements[idx] assert copied.final_measurement is not None measurement.metrics[name] = copied.final_measurement.metrics[ - previous_name + exptr_metric_name ] for suggestion, measurement in zip(suggestions, measurements): diff --git a/vizier/benchmarks/experimenters/__init__.py b/vizier/benchmarks/experimenters/__init__.py index 982d43531..412bd7b24 100644 --- a/vizier/benchmarks/experimenters/__init__.py +++ b/vizier/benchmarks/experimenters/__init__.py @@ -25,6 +25,7 @@ from vizier._src.benchmarks.experimenters.discretizing_experimenter import DiscretizingExperimenter from vizier._src.benchmarks.experimenters.experimenter import Experimenter from vizier._src.benchmarks.experimenters.experimenter_factory import BBOBExperimenterFactory +from vizier._src.benchmarks.experimenters.experimenter_factory import CombinedExperimenterFactory from vizier._src.benchmarks.experimenters.experimenter_factory import ExperimenterFactory from vizier._src.benchmarks.experimenters.experimenter_factory import SerializableExperimenterFactory from vizier._src.benchmarks.experimenters.experimenter_factory import SingleObjectiveExperimenterFactory