work on 2 fronts: 1. Clean up BayBE workflow into a single standalone…

… notebook 2. Correct range error/add options to set optimization parameter space bounds

run_experiment_baybe.py

@@ -53,22 +53,43 @@
 from src.schwefel import SchwefelProblem
 from time import time
 
-def run_experiment(n_init, noise_level, budget, seed, noise_bool):
+def run_experiment(n_init, noise_level, budget, seed, noise_bool, bounds):
+    """
+    Run a bayesian optimization campaign on the 2-dimensional
+    schwefel function using the specified parameters. Uses BayBE with the
+    SequentialGreedyRecommender with Expected improvement acquisition function.
+
+    :param n_init: Number of randomly selected initial trials to run
+    :type n_init: int
+    :param noise_level: Variance of Gaussian noise to add to scwhefel function values
+    :type noise_level: float
+    :param budget: Number of optimization trials to run (in addition to n_init)
+    :type budget: int
+    :param seed: Random seed
+    :type seed: int
+    :param noise_bool: Artifact from Botorch implementation, does nothing
+    :type noise_bool: bool
+    :return train_X: Scwhefel X values evaluated
+    :type train_X: Tensor
+    :return train_Y: The Schwefel function values associated with train_X points, including noise
+    :type train_Y: Tensor
+    :return train_Y_real: The 'true' noise-free Y values
+    :type train_Y_real: Tensor 
+    """
 
     N_DIMS_SCHWEF = 2
     ITERATION_BATCH_SIZE = 1
-
 
     torch.manual_seed(seed)
     np.random.seed(seed)
 
-    problem = SchwefelProblem(n_var=N_DIMS_SCHWEF, noise_level=noise_level)
+    problem = SchwefelProblem(n_var=N_DIMS_SCHWEF, noise_level=noise_level, range = bounds)
 
-    bounds = torch.tensor(problem.bounds, **tkwargs)
+    #bounds = torch.tensor(problem.bounds, **tkwargs)
 
     target = NumericalTarget(name = 'schwefel', mode = "MIN")
     parameters = [
-        NumericalContinuousParameter(f'schwefel{i+1}', bounds = (-50,50)) for i in range(N_DIMS_SCHWEF)
+        NumericalContinuousParameter(f'schwefel{i+1}', bounds = bounds) for i in range(N_DIMS_SCHWEF)
     ]
 
     objective = Objective(mode = "SINGLE", targets = [target])

run_experiment_random_baybe.py

@@ -0,0 +1,160 @@
+# %%
+import matplotlib.pyplot as plt
+import numpy as np
+import torch
+
+from botorch.models.gp_regression import (
+    SingleTaskGP,
+)
+from gpytorch.mlls.exact_marginal_log_likelihood import ExactMarginalLogLikelihood
+from botorch.fit import fit_gpytorch_model
+from botorch.models.transforms.outcome import Standardize
+
+from botorch.optim.optimize import optimize_acqf
+from botorch.acquisition.monte_carlo import qNoisyExpectedImprovement
+from botorch.sampling.normal import SobolQMCNormalSampler
+from botorch.utils.transforms import normalize, unnormalize
+import os
+import gc
+
+
+
+from baybe.targets import NumericalTarget
+from baybe.objective import Objective
+from baybe.parameters import (
+    NumericalContinuousParameter
+)
+
+from baybe.recommenders import (
+    SequentialGreedyRecommender,
+    RandomRecommender
+)
+
+from baybe.searchspace import SearchSpace
+from baybe import Campaign
+from baybe import simulation
+
+tkwargs = {
+    "dtype": torch.double,
+    "device": torch.device("cuda" if torch.cuda.is_available() else "cpu"),
+}
+SMOKE_TEST = os.environ.get("SMOKE_TEST")
+# SMOKE_TEST = True
+print("SMOKE_TEST", SMOKE_TEST)
+NUM_RESTARTS = 10 if not SMOKE_TEST else 2
+RAW_SAMPLES = 512 if not SMOKE_TEST else 4
+MC_SAMPLES = 128 if not SMOKE_TEST else 16
+batch_size = 1
+
+
+
+# %%
+from botorch.utils.sampling import draw_sobol_samples
+from src.schwefel import SchwefelProblem
+from time import time
+
+def run_experiment(n_init, noise_level, budget, seed, noise_bool, bounds):
+    """
+    Run a bayesian optimization campaign on the 2-dimensional
+    schwefel function using the specified parameters. Uses BayBE with the
+    SequentialGreedyRecommender with Expected improvement acquisition function.
+
+    :param n_init: Number of randomly selected initial trials to run
+    :type n_init: int
+    :param noise_level: Variance of Gaussian noise to add to scwhefel function values
+    :type noise_level: float
+    :param budget: Number of optimization trials to run (in addition to n_init)
+    :type budget: int
+    :param seed: Random seed
+    :type seed: int
+    :param noise_bool: Artifact from Botorch implementation, does nothing
+    :type noise_bool: bool
+    :return train_X: Scwhefel X values evaluated
+    :type train_X: Tensor
+    :return train_Y: The Schwefel function values associated with train_X points, including noise
+    :type train_Y: Tensor
+    :return train_Y_real: The 'true' noise-free Y values
+    :type train_Y_real: Tensor 
+    """
+
+    N_DIMS_SCHWEF = 2
+    ITERATION_BATCH_SIZE = 1
+
+
+    torch.manual_seed(seed)
+    np.random.seed(seed)
+
+    problem = SchwefelProblem(n_var=N_DIMS_SCHWEF, noise_level=noise_level, range = bounds)
+
+    #bounds = torch.tensor(problem.bounds, **tkwargs)
+
+    target = NumericalTarget(name = 'schwefel', mode = "MIN")
+    parameters = [
+        NumericalContinuousParameter(f'schwefel{i+1}', bounds = bounds) for i in range(N_DIMS_SCHWEF)
+    ]
+
+    objective = Objective(mode = "SINGLE", targets = [target])
+    searchspace = SearchSpace.from_product(parameters)
+
+
+    recommender_init = RandomRecommender()
+    #recommender_main = SequentialGreedyRecommender(acquisition_function_cls='EI')
+
+
+    print("Collecting random observations observations")
+    campaign_init = Campaign(searchspace, objective, recommender_init)
+    random_params = campaign_init.recommend(n_init + budget)
+
+    y_init = problem.y(random_params.to_numpy())
+    y_init_real = problem.f(random_params.to_numpy())
+
+    random_params.insert(N_DIMS_SCHWEF, 'schwefel', y_init)
+
+    #optimization_campaign = Campaign(searchspace, objective, recommender_main)
+    campaign_init.add_measurements(random_params)
+
+    #y_real = []
+    #print('Beginning optimization campaign')
+    #for i in range(budget):
+    #    reccs = optimization_campaign.recommend(ITERATION_BATCH_SIZE)
+   # 
+   #     y_vals = problem.y(reccs.to_numpy())
+   #     y_real.append(problem.f(reccs.to_numpy()))
+   # 
+   #     reccs.insert(N_DIMS_SCHWEF, 'schwefel', y_vals)
+   # 
+   #     optimization_campaign.add_measurements(reccs)
+
+    measurements = campaign_init.measurements
+
+    # get X and noisy y values
+    x_names = [f'schwefel{i+1}' for i in range(N_DIMS_SCHWEF)]
+    x_train = measurements[x_names].to_numpy()
+    y_train = measurements['schwefel'].to_numpy()
+
+    # compile noise-free ground truth vals
+    y_real_complete = y_init_real
+
+    #for i, val in enumerate(y_init_real):
+    #    y_real_complete[i] = val#
+
+    #for i, val in enumerate(y_real):
+    #    y_real_complete[i+len(y_init_real)] = val
+
+
+
+    train_X = torch.from_numpy(x_train)
+    train_Y = torch.from_numpy(y_train)
+    train_Y_real = torch.from_numpy(y_real_complete)
+
+    os.makedirs('results_random_baybe', exist_ok=True)
+    fname = f"results_random_baybe/{problem.__class__.__name__[:5]}_n_init_{n_init}_noiselvl_{noise_level}_budget_{budget}_seed_{seed}_noise_{noise_bool}.pt"
+    torch.save((train_X, train_Y, train_Y_real, None), fname)
+
+    return train_X, train_Y, train_Y_real, None
+
+
+
+if __name__ == "__main__":
+    run_experiment(5, 0.1, 5, 0, True)
+    run_experiment(5, 0.1, 5, 0, False)

run_grid_experiments_baybe.py

@@ -0,0 +1,60 @@
+#import ray
+import argparse
+from time import time, sleep
+from run_experiment_baybe import run_experiment
+from datetime import datetime
+import gc
+
+MAX_NUM_PENDING_TASKS = 12
+
+
+#@ray.remote
+def worker(n_init, noise_level, budget, seed, noise_bool, bounds):
+
+    try:
+        run_experiment(n_init, noise_level, budget, seed, noise_bool,bounds)
+        # saved file looks like this: results\Schwe_n_init_6_noiselvl_0_budget_0_seed_2_noise_False.pt
+    except Exception as e:
+        print(e)
+        print(f'problem {n_init} noise {noise_level} budget {budget} seed {seed} failed')
+        return 1
+
+    return 0
+
+def run_grid_experiments(seeds, n_inits, noise_levels, noise_bools, budget, bounds):
+
+    # ray.init(local_mode=True)
+    #ray.init(ignore_reinit_error=True)
+    start_time = time()
+    tasks = []
+
+    for seed in seeds:
+        for n_init in n_inits:
+            for noise_level in noise_levels:
+                for noise_bool in noise_bools:
+                    #if len(tasks) > MAX_NUM_PENDING_TASKS:
+                    #    completed_tasks, tasks = ray.wait(tasks, num_returns=1)
+                    #    ray.get(completed_tasks[0])
+
+                    #sleep(1)
+                    task = worker(n_init, noise_level, budget, seed, noise_bool, bounds)
+                    tasks.append(task)
+                    print(f'Started problem {n_init} noise {noise_level} budget {budget} seed {seed}, time: {time() - start_time:.2f}s')
+                    #gc.collect()
+
+   # while len(tasks) > 0:
+   #     completed_tasks, tasks = ray.wait(tasks, num_returns=1)
+   #     print(ray.get(completed_tasks[0]))
+
+
+    print('all experiments done, time: %.2fs' % (time() - start_time))
+
+if __name__ == "__main__":
+
+    seeds = [0]
+    n_inits = [2, 4, 6 ,8, 10]
+    noise_levels = [0, 0.01, 0.1, 0.5]
+    # budgets = [10, 20, 50]
+    noise_bools = [True, False]
+    budget = 10
+    run_grid_experiments(seeds, n_inits, noise_levels, noise_bools, budget)

run_grid_experiments_baybe_random.py

@@ -0,0 +1,60 @@
+#import ray
+import argparse
+from time import time, sleep
+from run_experiment_random_baybe import run_experiment
+from datetime import datetime
+import gc
+
+MAX_NUM_PENDING_TASKS = 12
+
+
+#@ray.remote
+def worker(n_init, noise_level, budget, seed, noise_bool, bounds):
+
+    try:
+        run_experiment(n_init, noise_level, budget, seed, noise_bool, bounds)
+        # saved file looks like this: results\Schwe_n_init_6_noiselvl_0_budget_0_seed_2_noise_False.pt
+    except Exception as e:
+        print(e)
+        print(f'problem {n_init} noise {noise_level} budget {budget} seed {seed} failed')
+        return 1
+
+    return 0
+
+def run_grid_experiments_random(seeds, n_inits, noise_levels, noise_bools, budget, bounds):
+
+    # ray.init(local_mode=True)
+    #ray.init(ignore_reinit_error=True)
+    start_time = time()
+    tasks = []
+
+    for seed in seeds:
+        for n_init in n_inits:
+            for noise_level in noise_levels:
+                for noise_bool in noise_bools:
+                    #if len(tasks) > MAX_NUM_PENDING_TASKS:
+                    #    completed_tasks, tasks = ray.wait(tasks, num_returns=1)
+                    #    ray.get(completed_tasks[0])
+
+                    #sleep(1)
+                    task = worker(n_init, noise_level, budget, seed, noise_bool, bounds)
+                    tasks.append(task)
+                    print(f'Started problem {n_init} noise {noise_level} budget {budget} seed {seed}, time: {time() - start_time:.2f}s')
+                    #gc.collect()
+
+   # while len(tasks) > 0:
+   #     completed_tasks, tasks = ray.wait(tasks, num_returns=1)
+   #     print(ray.get(completed_tasks[0]))
+
+
+    print('all experiments done, time: %.2fs' % (time() - start_time))
+
+if __name__ == "__main__":
+
+    seeds = [0]
+    n_inits = [2, 4, 6 ,8, 10]
+    noise_levels = [0, 0.01, 0.1, 0.5]
+    # budgets = [10, 20, 50]
+    noise_bools = [True, False]
+    budget = 10
+    run_grid_experiments(seeds, n_inits, noise_levels, noise_bools, budget)

src/schwefel.py

@@ -7,7 +7,7 @@ def __init__(self, n_var=1, noise_level=0.01, range = (-50, 50)):
         """
         self.noise_level = noise_level
         self.n_var = n_var  # Number of variables/dimensions
-        self.bounds = np.array([[-50] * self.n_var, [50] * self.n_var])
+        self.bounds = np.array([[range[0]] * self.n_var, [range[1]] * self.n_var])
 
     def _schwefel_individual(self, x):
         return x * np.sin(np.sqrt(np.abs(x)))