add irace

DESCRIPTION

@@ -50,6 +50,7 @@ Imports:
 Suggests:
  adagio,
  GenSA,
+ irace,
  mlr3pipelines,
  nloptr,
  rpart,
@@ -71,13 +72,15 @@ Collate:
  'TunerFromOptimizer.R'
  'TunerGenSA.R'
  'TunerGridSearch.R'
+ 'TunerIrace.R'
  'TunerNLoptr.R'
  'TunerRandomSearch.R'
  'TuningInstanceMulticrit.R'
  'TuningInstanceSingleCrit.R'
  'assertions.R'
  'bibentries.R'
  'helper.R'
+ 'irace_helpers.R'
  'reexport.R'
  'sugar.R'
  'zzz.R'

NAMESPACE

@@ -11,6 +11,7 @@ export(TunerDesignPoints)
 export(TunerFromOptimizer)
 export(TunerGenSA)
 export(TunerGridSearch)
+export(TunerIrace)
 export(TunerNLoptr)
 export(TunerRandomSearch)
 export(TuningInstanceMultiCrit)

NEWS.md

@@ -1,5 +1,6 @@
 # mlr3tuning 0.7.0.9000
 
+- Adds `TunerIrace` from `irace` package.
 - Adds `tune()`, `auto_tuner()` and `tune_nested()` sugar functions.
 - `TuningInstanceSingleCrit`, `TuningInstanceMultiCrit` and `AutoTuner` can be
  initialized with `store_benchmark_result = FALSE` and `store_models = TRUE`
@@ -19,7 +20,7 @@
 
 # mlr3tuning 0.5.0
 
-- Adds `TunerCmaes` from adagio package.
+- Adds `TunerCmaes` from `adagio` package.
 - Fix `predict_type` in `AutoTuner`.
 - Support to set `TuneToken` in `Learner$param_set` and create a search space
  from it.

R/ObjectiveTuning.R

@@ -24,9 +24,6 @@ ObjectiveTuning = R6Class("ObjectiveTuning",
  #' @field learner ([mlr3::Learner]).
  learner = NULL,
 
- #' @field resampling ([mlr3::Resampling]).
- resampling = NULL,
-
  #' @field measures (list of [mlr3::Measure]).
  measures = NULL,
 
@@ -47,16 +44,11 @@ ObjectiveTuning = R6Class("ObjectiveTuning",
 
  self$task = assert_task(as_task(task, clone = TRUE))
  self$learner = assert_learner(as_learner(learner, clone = TRUE))
- self$resampling = assert_resampling(as_resampling(
- resampling,
- clone = TRUE))
+ self$resampling = resampling
  self$measures = assert_measures(as_measures(measures, clone = TRUE),
  task = self$task, learner = self$learner)
  self$store_benchmark_result = assert_logical(store_benchmark_result)
  self$store_models = assert_logical(store_models)
- if (!resampling$is_instantiated) {
- self$resampling$instantiate(self$task)
- }
 
  codomain = ParamSet$new(map(self$measures, function(s) {
  ParamDbl$new(id = s$id, tags = ifelse(s$minimize, "minimize", "maximize"))
@@ -90,6 +82,22 @@ ObjectiveTuning = R6Class("ObjectiveTuning",
  } else {
  aggr[, y, with = FALSE]
  }
+ },
+
+ .resampling = NULL
+ ),
+
+ active = list(
+
+ #' @field resampling ([mlr3::Resampling]).
+ resampling = function(rhs) {
+ if(missing(rhs)) {
+ private$.resampling
+ } else {
+ resampling = assert_resampling(as_resampling(rhs, clone = TRUE))
+ if (!resampling$is_instantiated) resampling$instantiate(self$task)
+ private$.resampling = resampling
+ }
  }
  )
 )

R/TunerIrace.R

@@ -0,0 +1,153 @@
+#' @title Tuning via Iterated Racing.
+#'
+#' @include Tuner.R
+#' @name mlr_tuners_irace
+#'
+#' @description
+#' `TunerIrace` class that implements iterated racing. Calls [irace::irace()]
+#' from package \CRANpkg{irace}.
+#'
+#' @section Parameters:
+#' \describe{
+#' \item{`n_instances`}{`integer(1)`\cr
+#' Number of resampling instances.}
+#' }
+#'
+#' For the meaning of all other parameters, see [irace::defaultScenario()]. Note
+#' that we have removed all control parameters which refer to the termination of
+#' the algorithm. Use [TerminatorRunTime] or [TerminatorEvals] instead. Other
+#' terminators do not work with `TunerIrace`. We substract 5 seconds from the
+#' [TerminatorRunTime] budget for stability reasons.
+#'
+#' @templateVar id irace
+#' @template section_dictionary_tuners
+#'
+#' @source
+#' `r format_bib("lopez_2016")`
+#'
+#' @family Tuner
+#' @export
+#' @examples
+#' library(mlr3)
+#' library(paradox)
+#' search_space = ParamSet$new(list(
+#' ParamDbl$new("cp", lower = 0.001, upper = 0.1)
+#' ))
+#' terminator = trm("evals", n_evals = 42)
+#' instance = TuningInstanceSingleCrit$new(
+#' task = tsk("iris"),
+#' learner = lrn("classif.rpart"),
+#' resampling = rsmp("holdout"),
+#' measure = msr("classif.ce"),
+#' search_space = search_space,
+#' terminator = terminator
+#' )
+#' tt = tnr("irace")
+#'
+#' # modifies the instance by reference
+#' tt$optimize(instance)
+#'
+#' # returns best configuration and best performance
+#' instance$result
+#'
+#' # allows access of data.table of full path of all evaluations
+#' instance$archive
+TunerIrace = R6Class("TunerIrace",
+ inherit = Tuner,
+ public = list(
+ #' @description
+ #' Creates a new instance of this [R6][R6::R6Class] class.
+ initialize = function() {
+ ps = ParamSet$new(list(
+ ParamInt$new("n_instances", lower = 1, default = 10),
+ ParamInt$new("debugLevel", default = 0, lower = 0),
+ ParamInt$new("seed"),
+ ParamDbl$new("postselection", default = 0, lower = 0, upper = 1),
+ ParamInt$new("elitist", default = 1, lower = 0, upper = 1),
+ ParamInt$new("elitistLimit", default = 2, lower = 0),
+ ParamInt$new("nbIterations", default = 0, lower = 0),
+ ParamInt$new("nbExperimentsPerIteration", default = 0, lower = 0),
+ ParamInt$new("minNbSurvival", default = 0, lower = 0),
+ ParamInt$new("nbConfigurations", default = 0, lower = 0),
+ ParamInt$new("mu", default = 5, lower = 1),
+ ParamInt$new("softRestart", default = 1, lower = 0, upper = 1),
+ ParamDbl$new("softRestartThreshold"),
+ ParamInt$new("digits", default = 4, lower = 1, upper = 15),
+ ParamFct$new("testType", default = "F-test",
+ levels = c("F-test", "t-test", "t-test-bonferroni", "t-test-holm")),
+ ParamInt$new("firstTest", default = 5, lower = 0),
+ ParamInt$new("eachTest", default = 1, lower = 1),
+ ParamDbl$new("confidence", default = 0.95, lower = 0, upper = 1),
+ ParamInt$new("capping", default = 0, lower = 0, upper = 1),
+ ParamFct$new("cappingType", default = "median", levels = c("median", "mean", "best", "worst")),
+ ParamFct$new("boundType", default = "candidate", levels = c("candidate", "instance")),
+ ParamDbl$new("boundMax", default = 0),
+ ParamInt$new("boundDigits", default = 0),
+ ParamDbl$new("boundPar", default = 1),
+ ParamDbl$new("boundAsTimeout", default = 1)
+ ))
+ ps$values = list(n_instances = 10)
+
+ super$initialize(
+ param_set = ps,
+ param_classes = c("ParamDbl", "ParamInt", "ParamFct", "ParamLgl"),
+ properties = c("dependencies", "single-crit"),
+ packages = "irace"
+ )
+ }
+ ),
+
+ private = list(
+ .optimize = function(inst) {
+ pv = self$param_set$values
+ terminator = inst$terminator
+ objective = inst$objective
+
+ # Check terminators 
+ if (!(inherits(terminator, "TerminatorEvals") || inherits(terminator, "TerminatorRunTime"))) {
+ stopf("%s is not supported. Use <TerminatorEvals> or <TerminatorRunTime> instead.", format(inst$terminator))
+ }
+
+ # Set resampling instances
+ ri = replicate(pv$n_instances, {
+ r = objective$resampling$clone()
+ r$instantiate(objective$task)
+ })
+ pv$n_instances = NULL
+
+ # Make scenario
+ scenario = c(list(
+ targetRunner = target_runner,
+ logFile = tempfile(),
+ instances = ri,
+ debugLevel = 0,
+ maxExperiments = if (inherits(terminator, "TerminatorEvals")) terminator$param_set$values$n_evals else 0,
+ maxTime = if (inherits(terminator, "TerminatorRunTime")) terminator$param_set$values$secs - 5 else 0,
+ targetRunnerData = list(inst = inst)
+ ), pv)
+
+ res = irace::irace(scenario = scenario, parameters = paradox_to_irace(inst$search_space))
+
+ # Temporarily store result
+ private$.result_id = res$.ID.[1]
+ },
+
+ # The final configurations returned by irace are the elites of the final race.
+ # We store the best performing one.
+ # The reported performance value is the average of all resampling iterations.
+ .assign_result = function(inst) {
+ if(length(private$.result_id) == 0) {
+ stop("irace::irace did not return a result. The evaluated configurations are still accessible through the archive.")
+ }
+ res = inst$archive$data[get("id_configuration") == private$.result_id, ]
+ cols = c(inst$archive$cols_x, "id_configuration")
+ xdt = res[1, cols, with = FALSE]
+ y = set_names(mean(unlist(res[, inst$archive$cols_y, with = FALSE])), inst$archive$cols_y)
+ inst$assign_result(xdt, y)
+ },
+
+ .result_id = NULL
+ )
+)
+
+mlr_tuners$add("irace", TunerIrace)

R/bibentries.R

@@ -12,7 +12,6 @@ bibentries = c(
  doi = "10.32614/rj-2013-002"
  ),
 
-
  tsallis_1996 = bibentry("article",
  title = "Generalized simulated annealing",
  author = "Constantino Tsallis and Daniel A. Stariolo",
@@ -42,5 +41,15 @@ bibentries = c(
  author = "Johnson, Steven G",
  url = "https://github.com/stevengj/nlopt",
  year = "2020"
+ ),
+
+ lopez_2016 = bibentry("article",
+ title = "The irace package: Iterated racing for automatic algorithm configuration",
+ author = "Manuel Lopez-Ibanez and Jeremie Dubois-Lacoste and Leslie Perez Caceres and Mauro Birattari and Thomas Stuetzle",
+ year = "2016",
+ journal = "Operations Research Perspectives",
+ volume = "3",
+ pages = "43--58",
+ doi = "https://doi.org/10.1016/j.orp.2016.09.002"
  )
 )

R/irace_helpers.R

@@ -0,0 +1,84 @@
+paradox_to_irace = function(ps) {
+
+ assertClass(ps, "ParamSet")
+ if("ParamUty" %in% ps$class) {
+ stop("<ParamUty> not supported by <TunerIrace>")
+ }
+
+ # what about ParamUty = vector numeric/real
+ class_lookup = data.table(
+ paradox = c("ParamLgl", "ParamInt", "ParamDbl", "ParamFct"),
+ irace = c("c", "i", "r", "c"), stringsAsFactors = FALSE)
+
+ type = unlist(subset(merge(data.table(paradox = ps$class), class_lookup, sort = FALSE), select = "irace"))
+ range = get_irace_range(ps)
+ if (ps$has_deps) {
+ condition = get_irace_condition(ps)
+ } else {
+ condition = NULL
+ }
+
+ par_tab = paste(ps$ids(), '""', type, range, condition$cond, collapse = "\n")
+
+ return(irace::readParameters(text = par_tab))
+}
+
+get_irace_range = function(ps) {
+ rng = data.table(lower = ps$lower, upper = ps$upper, lvl = ps$levels)
+
+ apply(rng, 1, function(x) {
+ if (is.na(x[[1]])) {
+ return(paste0("(", paste0(x[[3]], collapse = ","), ")"))
+ } else {
+ return(paste0("(", x[[1]], ",", x[[2]], ")"))
+ }
+ })
+}
+
+get_irace_condition = function(ps) {
+ cond = rbindlist(apply(ps$deps, 1, function(x) {
+ on = x[[2]]
+ cond = x[[3]]$rhs
+ if (is.character(cond)) {
+ cond = paste0("'", cond, "'")
+ }
+ if (x[[3]]$type == "equal") {
+ condition = paste("|", x[[2]], "==", cond)
+ } else {
+ condition = paste("|", x[[2]], "%in%", paste0("c(", paste0(cond, collapse = ","), ")"))
+ }
+ data.table(id = x[[1]], cond = condition)
+ }))
+
+ # coercion back and forth from frame/table is due to data.frame sorting even when sort = FALSE
+ tab = data.frame(merge(data.table(id = ps$ids()), cond, by = "id", all.x = TRUE, sort = FALSE))
+ tab[is.na(tab)] = ""
+
+ return(tab)
+}
+
+target_runner = function(experiment, scenario) { # nolint
+ t0 = Sys.time()
+ tuning_instance = scenario$targetRunnerData$inst
+
+ # fix logicals
+ config = as.data.table(lapply(experiment$configuration, function(x) {
+ if (x %in% c("TRUE", "FALSE")) {
+ return(as.logical(x))
+ } else {
+ return(x)
+ }
+ }))
+
+ # change resampling instance
+ tuning_instance$objective$resampling = experiment$instance
+
+ # add extra info to archive
+ extra = data.table(id_configuration = experiment$id.configuration, id_instance = experiment$id.instance)
+
+ # evaluate configuration
+ # objective_function cannot pass extra information
+ cost = as.numeric(tuning_instance$eval_batch(cbind(config, extra))) * tuning_instance$objective_multiplicator
+
+ return(list(cost = cost, time = as.numeric(difftime(Sys.time(), t0, units = "secs"))))
+}

inst/testthat/helper_expectations.R

@@ -44,3 +44,36 @@ expect_irace_parameters = function(parameters, names, types, domain, conditions,
  expect_equal(parameters$nbFixed, 0)
  expect_equal(parameters$nbVariable, length(names))
 }
+
+expect_irace_parameters = function(parameters, names, types, domain, conditions, depends,
+ hierarchy) {
+ expect_list(parameters, len = 12, any.missing = FALSE)
+ expect_equal(names(parameters), c("names", "types", "switches", "domain", "conditions", "isFixed",
+ "transform", "depends", "hierarchy", "nbParameters", "nbFixed",
+ "nbVariable"))
+ expect_equal(parameters$names, names)
+ expect_equal(parameters$types, set_names(types, names))
+ expect_equal(parameters$switches, named_vector(names, ""))
+ expect_equal(parameters$domain, domain)
+ if (missing(conditions)) {
+ expect_equal(parameters$conditions, named_list(names, TRUE))
+ } else {
+ # can't compare expressions directly
+ expect_equal(as.character(parameters$conditions), as.character(conditions))
+ }
+ expect_equal(parameters$isFixed, named_vector(names, FALSE))
+ expect_equal(parameters$transform, named_list(names, ""))
+ if (missing(depends)) {
+ expect_equal(parameters$depends, named_list(names, character(0)))
+ } else {
+ expect_equal(parameters$depends, depends)
+ }
+ if (missing(hierarchy)) {
+ expect_equal(parameters$hierarchy, named_vector(names, 1))
+ } else {
+ expect_equal(parameters$hierarchy, set_names(hierarchy, names))
+ }
+ expect_equal(parameters$nbParameters, length(names))
+ expect_equal(parameters$nbFixed, 0)
+ expect_equal(parameters$nbVariable, length(names))
+}