From ef51f1022c85a2909db958350b08de491ca79445 Mon Sep 17 00:00:00 2001 From: Raphael Date: Mon, 9 Mar 2020 19:37:27 +0000 Subject: [PATCH 01/40] add irace --- DESCRIPTION | 3 + NAMESPACE | 1 + R/TunerIrace.R | 94 +++++++++++++++++++++ R/irace_helpers.R | 64 ++++++++++++++ man-roxygen/section_dictionary_tuners.R | 1 + man/Tuner.Rd | 1 + man/mlr_tuners.Rd | 1 + man/mlr_tuners_design_points.Rd | 4 +- man/mlr_tuners_gensa.Rd | 4 +- man/mlr_tuners_grid_search.Rd | 4 +- man/mlr_tuners_irace.Rd | 107 ++++++++++++++++++++++++ man/mlr_tuners_random_search.Rd | 4 +- tests/testthat/test_TunerIrace.R | 36 ++++++++ 13 files changed, 320 insertions(+), 4 deletions(-) create mode 100644 R/TunerIrace.R create mode 100644 R/irace_helpers.R create mode 100644 man/mlr_tuners_irace.Rd create mode 100644 tests/testthat/test_TunerIrace.R diff --git a/DESCRIPTION b/DESCRIPTION index 14e0ce1fc..851f6cadd 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -43,6 +43,7 @@ Imports: R6 Suggests: GenSA, + irace, mlr3pipelines, rpart, testthat @@ -66,9 +67,11 @@ Collate: 'TunerDesignPoints.R' 'TunerGenSA.R' 'TunerGridSearch.R' + 'TunerIrace.R' 'TunerRandomSearch.R' 'TuningInstance.R' 'assertions.R' 'helper.R' + 'irace_helpers.R' 'sugar.R' 'zzz.R' diff --git a/NAMESPACE b/NAMESPACE index 98dccd889..231bd435b 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -13,6 +13,7 @@ export(Tuner) export(TunerDesignPoints) export(TunerGenSA) export(TunerGridSearch) +export(TunerIrace) export(TunerRandomSearch) export(TuningInstance) export(mlr_terminators) diff --git a/R/TunerIrace.R b/R/TunerIrace.R new file mode 100644 index 000000000..e831f720f --- /dev/null +++ b/R/TunerIrace.R @@ -0,0 +1,94 @@ +#' @title TunerIrace +#' +#' @name mlr_tuners_irace +#' @include Tuner.R +#' +#' @description +#' Subclass for iterated racing tuning calling [irace::irace()] from package \CRANpkg{irace}. +#' +#' @section Parameters: +#' * `show.irace.output` (`logical(1)`) +#' * `debugLevel` (`integer(1)`) +#' * `seed` (`integer(1)`) +#' * `postselection` (`numeric(1)`) +#' * `elitist` (`integer(1)`) +#' * `elitistLimit` (`integer(1)`) +#' * `mu` (`integer(1)`) +#' * `softRestart` (`integer(1)`) +#' * `softRestartThreshold` (`numeric(1)`) +#' * `digits` (`integer(1)`) +#' * `testType` (`character(1)`) +#' * `firstTest` (`integer(1)`) +#' * `eachTest` (`integer(1)`) +#' * `confidence` (`numeric(1)`) +#' * `capping` (`integer(1)`) +#' * `cappingType` (`character(1)`) +#' * `boundType` (`character(1)`) +#' * `boundMax` (`numeric(1)`) +#' * `boundDigits` (`integer(1)`) +#' * `boundPar` (`numeric(1)`) +#' * `boundAsTimeout` (`numeric(1)`) +#' +#' `show.irace.output` suppresses the output from [irace::irace()] and only prints output from this package. +#' For the meaning of all other parameters, see [irace::defaultScenario()]. +#' Note that we have removed all control parameters which refer to the termination of the algorithm and +#' where our terminators allow to obtain the same behavior. +#' +#' @templateVar id irace +#' @template section_dictionary_tuners +#' +#' @family Tuner +#' @export +#' @examples +#' # see ?Tuner +TunerIrace = R6Class("TunerIrace", inherit = Tuner, + public = list( + #' @description + #' Creates a new instance of this [R6][R6::R6Class] class. + initialize = function() { + ps = ParamSet$new(list( + ParamLgl$new("show.irace.output", default = FALSE), + ParamInt$new("debugLevel", default = 0, lower = 0), + ParamInt$new("seed"), + ParamDbl$new("postselection",default=0,lower=0,upper=1), + ParamInt$new("elitist",default=1,lower=0,upper=1), + ParamInt$new("elitistLimit",default=2,lower=0), + ParamInt$new("mu",default=5,lower=1), + ParamInt$new("softRestart",default=1,lower=0,upper=1), + ParamDbl$new("softRestartThreshold"), + ParamInt$new("digits", default = 4, lower = 0), + ParamFct$new("testType", default = "F-test", levels = c("F-test","t-test","t-test-bonferroni","t-test-holm")), + ParamInt$new("firstTest", default = 5, lower = 0), + ParamInt$new("eachTest",default = 1, lower = 1), + ParamDbl$new("confidence",default=0.95,lower=0,upper=1), + ParamInt$new("capping",default = 0,lower=0,upper=1), + ParamFct$new("cappingType",default="median",levels=c("median","mean","best","worst")), + ParamFct$new("boundType",default="candidate",levels=c("candidate","instance")), + ParamDbl$new("boundMax",default=0), + ParamInt$new("boundDigits",default=0), + ParamDbl$new("boundPar",default=1), + ParamDbl$new("boundAsTimeout",default=1) + )) + + ps$values$show.irace.output = FALSE + + super$initialize( + param_set = ps, + param_classes = c("ParamDbl","ParamInt","ParamFct","ParamLgl"), + properties = character(), + packages = "irace" + ) + } + ), + + private = list( + .tune = function(instance) { + ps = self$param_set$values[!(names(self$param_set$values) %in% c("show.irace.output"))] + g = if (self$param_set$values$show.irace.output) identity else capture.output # copied from mlr + g(irace::irace(scenario = c(makeScenario(instance), ps), + parameters = paradoxToIrace(instance$param_set))) + } + ) +) + +mlr_tuners$add("irace", TunerIrace) diff --git a/R/irace_helpers.R b/R/irace_helpers.R new file mode 100644 index 000000000..ca8c837ee --- /dev/null +++ b/R/irace_helpers.R @@ -0,0 +1,64 @@ +paradoxToIrace = function(ps){ + assertClass(ps, "ParamSet") + # what about ParamUty = vector numeric/real + class_lookup = data.frame(paradox = c("ParamLgl","ParamInt","ParamDbl","ParamFct"), + irace = c("c","i","r","c"), stringsAsFactors = FALSE) + name <- ps$ids() + type <- class_lookup[match(ps$class, class_lookup$paradox), 2] + range <- getIraceRange(ps) + if(ps$has_deps){ + condition <- getIraceCondition(ps) + } else { + condition = NULL + } + + # does mlr3tuning deal with trafo internally or passed to irace? irace only handle log + par.tab <- paste(name, '""', type, range, condition, collapse = "\n") + irace::readParameters(text = par.tab) +} +getIraceRange = function(ps){ + rng = data.table(lower = ps$lower, upper = ps$upper, lvl = ps$levels) + + apply(rng, 1, function(x){ + if(is.na(x[[1]])) + return(paste0("(",paste0(x[[3]], collapse = ","),")")) + else + return(paste0("(",x[[1]],",",x[[2]],")")) + }) +} +getIraceCondition = function(ps){ + cond = apply(ps$deps, 1, function(x){ + if(class(x[[3]])[1] == "CondEqual"){ + condition = paste("|",x[[2]], "==", x[[3]]$rhs) + } else { + condition = paste("|",x[[2]],"%in%", paste0("c(",paste0(x[[3]]$rhs, collapse = ","),")")) + } + }) + + condition = data.table(id = ps$ids(), cond = "") + ind = match(ps$deps$id, condition$id) + condition$cond[ind] = cond + condition$cond +} + +makeScenario = function(instance){ + list( + targetRunner = targetRunner, + logFile = tempfile(), + instances = list(instance), + debugLevel = 0, + maxExperiments = ifelse(class(instance$terminator)[1] == "TerminatorEvals", + instance$terminator$param_set$values$n_evals, + 0), + maxTime = ifelse(class(instance$terminator)[1] == "TerminatorClockTime", + instance$terminator$param_set$values$secs, + 0) + ) +} + +targetRunner = function(experiment, scenario){ + t0 = Sys.time() + cost = scenario$instances[[1]]$eval_batch(as.data.table(experiment$configuration))$perf + t1 = Sys.time() + list(cost = cost, time = as.numeric(t1-t0)) +} diff --git a/man-roxygen/section_dictionary_tuners.R b/man-roxygen/section_dictionary_tuners.R index cf661623a..44bcc18a8 100644 --- a/man-roxygen/section_dictionary_tuners.R +++ b/man-roxygen/section_dictionary_tuners.R @@ -1,6 +1,7 @@ #' @section Dictionary: #' This [Tuner] can be instantiated via the [dictionary][mlr3misc::Dictionary] [mlr_tuners] or with the associated sugar function [tnr()]: #' ``` +#' <%= class(tnr(id))[1] %>$new() #' mlr_tuners$get("<%= id %>") #' tnr("<%= id %>") #' ``` diff --git a/man/Tuner.Rd b/man/Tuner.Rd index 74ebcf7d4..44eca0f61 100644 --- a/man/Tuner.Rd +++ b/man/Tuner.Rd @@ -78,6 +78,7 @@ Other Tuner: \code{\link{mlr_tuners_design_points}}, \code{\link{mlr_tuners_gensa}}, \code{\link{mlr_tuners_grid_search}}, +\code{\link{mlr_tuners_irace}}, \code{\link{mlr_tuners_random_search}}, \code{\link{mlr_tuners}} } diff --git a/man/mlr_tuners.Rd b/man/mlr_tuners.Rd index dcfa74eee..9b3f329eb 100644 --- a/man/mlr_tuners.Rd +++ b/man/mlr_tuners.Rd @@ -30,6 +30,7 @@ Other Tuner: \code{\link{mlr_tuners_design_points}}, \code{\link{mlr_tuners_gensa}}, \code{\link{mlr_tuners_grid_search}}, +\code{\link{mlr_tuners_irace}}, \code{\link{mlr_tuners_random_search}} } \concept{Tuner} diff --git a/man/mlr_tuners_design_points.Rd b/man/mlr_tuners_design_points.Rd index 832ddca42..c448c3c08 100644 --- a/man/mlr_tuners_design_points.Rd +++ b/man/mlr_tuners_design_points.Rd @@ -17,7 +17,8 @@ of termination criteria. } \section{Dictionary}{ -This \link{Tuner} can be instantiated via the \link[mlr3misc:Dictionary]{dictionary} \link{mlr_tuners} or with the associated sugar function \code{\link[=tnr]{tnr()}}:\preformatted{mlr_tuners$get("design_points") +This \link{Tuner} can be instantiated via the \link[mlr3misc:Dictionary]{dictionary} \link{mlr_tuners} or with the associated sugar function \code{\link[=tnr]{tnr()}}:\preformatted{TunerDesignPoints$new() +mlr_tuners$get("design_points") tnr("design_points") } } @@ -40,6 +41,7 @@ Other Tuner: \code{\link{Tuner}}, \code{\link{mlr_tuners_gensa}}, \code{\link{mlr_tuners_grid_search}}, +\code{\link{mlr_tuners_irace}}, \code{\link{mlr_tuners_random_search}}, \code{\link{mlr_tuners}} } diff --git a/man/mlr_tuners_gensa.Rd b/man/mlr_tuners_gensa.Rd index b3d48ed92..8df502c92 100644 --- a/man/mlr_tuners_gensa.Rd +++ b/man/mlr_tuners_gensa.Rd @@ -24,7 +24,8 @@ where our terminators allow to obtain the same behavior. \section{Dictionary}{ -This \link{Tuner} can be instantiated via the \link[mlr3misc:Dictionary]{dictionary} \link{mlr_tuners} or with the associated sugar function \code{\link[=tnr]{tnr()}}:\preformatted{mlr_tuners$get("gensa") +This \link{Tuner} can be instantiated via the \link[mlr3misc:Dictionary]{dictionary} \link{mlr_tuners} or with the associated sugar function \code{\link[=tnr]{tnr()}}:\preformatted{TunerGenSA$new() +mlr_tuners$get("gensa") tnr("gensa") } } @@ -37,6 +38,7 @@ Other Tuner: \code{\link{Tuner}}, \code{\link{mlr_tuners_design_points}}, \code{\link{mlr_tuners_grid_search}}, +\code{\link{mlr_tuners_irace}}, \code{\link{mlr_tuners_random_search}}, \code{\link{mlr_tuners}} } diff --git a/man/mlr_tuners_grid_search.Rd b/man/mlr_tuners_grid_search.Rd index a2d3f0213..26daa8824 100644 --- a/man/mlr_tuners_grid_search.Rd +++ b/man/mlr_tuners_grid_search.Rd @@ -18,7 +18,8 @@ of termination criteria. } \section{Dictionary}{ -This \link{Tuner} can be instantiated via the \link[mlr3misc:Dictionary]{dictionary} \link{mlr_tuners} or with the associated sugar function \code{\link[=tnr]{tnr()}}:\preformatted{mlr_tuners$get("grid_search") +This \link{Tuner} can be instantiated via the \link[mlr3misc:Dictionary]{dictionary} \link{mlr_tuners} or with the associated sugar function \code{\link[=tnr]{tnr()}}:\preformatted{TunerGridSearch$new() +mlr_tuners$get("grid_search") tnr("grid_search") } } @@ -43,6 +44,7 @@ Other Tuner: \code{\link{Tuner}}, \code{\link{mlr_tuners_design_points}}, \code{\link{mlr_tuners_gensa}}, +\code{\link{mlr_tuners_irace}}, \code{\link{mlr_tuners_random_search}}, \code{\link{mlr_tuners}} } diff --git a/man/mlr_tuners_irace.Rd b/man/mlr_tuners_irace.Rd new file mode 100644 index 000000000..52f74c3dd --- /dev/null +++ b/man/mlr_tuners_irace.Rd @@ -0,0 +1,107 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/TunerIrace.R +\name{mlr_tuners_irace} +\alias{mlr_tuners_irace} +\alias{TunerIrace} +\title{TunerIrace} +\description{ +Subclass for iterated racing tuning calling \code{\link[irace:irace]{irace::irace()}} from package \CRANpkg{irace}. +} +\section{Parameters}{ + +\itemize{ +\item \code{show.irace.output} (\code{logical(1)}) +\item \code{debugLevel} (\code{integer(1)}) +\item \code{seed} (\code{integer(1)}) +\item \code{postselection} (\code{numeric(1)}) +\item \code{elitist} (\code{integer(1)}) +\item \code{elitistLimit} (\code{integer(1)}) +\item \code{mu} (\code{integer(1)}) +\item \code{softRestart} (\code{integer(1)}) +\item \code{softRestartThreshold} (\code{numeric(1)}) +\item \code{digits} (\code{integer(1)}) +\item \code{testType} (\code{character(1)}) +\item \code{firstTest} (\code{integer(1)}) +\item \code{eachTest} (\code{integer(1)}) +\item \code{confidence} (\code{numeric(1)}) +\item \code{capping} (\code{integer(1)}) +\item \code{cappingType} (\code{character(1)}) +\item \code{boundType} (\code{character(1)}) +\item \code{boundMax} (\code{numeric(1)}) +\item \code{boundDigits} (\code{integer(1)}) +\item \code{boundPar} (\code{numeric(1)}) +\item \code{boundAsTimeout} (\code{numeric(1)}) +} + +\code{show.irace.output} suppresses the output from \code{\link[irace:irace]{irace::irace()}} and only prints output from this package. +For the meaning of all other parameters, see \code{\link[irace:defaultScenario]{irace::defaultScenario()}}. +Note that we have removed all control parameters which refer to the termination of the algorithm and +where our terminators allow to obtain the same behavior. +} + +\section{Dictionary}{ + +This \link{Tuner} can be instantiated via the \link[mlr3misc:Dictionary]{dictionary} \link{mlr_tuners} or with the associated sugar function \code{\link[=tnr]{tnr()}}:\preformatted{TunerIrace$new() +mlr_tuners$get("irace") +tnr("irace") +} +} + +\examples{ +# see ?Tuner +} +\seealso{ +Other Tuner: +\code{\link{Tuner}}, +\code{\link{mlr_tuners_design_points}}, +\code{\link{mlr_tuners_gensa}}, +\code{\link{mlr_tuners_grid_search}}, +\code{\link{mlr_tuners_random_search}}, +\code{\link{mlr_tuners}} +} +\concept{Tuner} +\section{Super class}{ +\code{\link[mlr3tuning:Tuner]{mlr3tuning::Tuner}} -> \code{TunerIrace} +} +\section{Methods}{ +\subsection{Public methods}{ +\itemize{ +\item \href{#method-new}{\code{TunerIrace$new()}} +\item \href{#method-clone}{\code{TunerIrace$clone()}} +} +} +\if{html}{ +\out{
Inherited methods} +\itemize{ +\item \out{}\href{../../mlr3tuning/html/Tuner.html#method-format}{\code{mlr3tuning::Tuner$format()}}\out{} +\item \out{}\href{../../mlr3tuning/html/Tuner.html#method-print}{\code{mlr3tuning::Tuner$print()}}\out{} +\item \out{}\href{../../mlr3tuning/html/Tuner.html#method-tune}{\code{mlr3tuning::Tuner$tune()}}\out{} +} +\out{
} +} +\if{html}{\out{
}} +\if{html}{\out{}} +\subsection{Method \code{new()}}{ +Creates a new instance of this \link[R6:R6Class]{R6} class. +\subsection{Usage}{ +\if{html}{\out{
}}\preformatted{TunerIrace$new()}\if{html}{\out{
}} +} + +} +\if{html}{\out{
}} +\if{html}{\out{}} +\subsection{Method \code{clone()}}{ +The objects of this class are cloneable with this method. +\subsection{Usage}{ +\if{html}{\out{
}}\preformatted{TunerIrace$clone(deep = FALSE)}\if{html}{\out{
}} +} + +\subsection{Arguments}{ +\if{html}{\out{
}} +\describe{ +\item{\code{deep}}{Whether to make a deep clone.} +} +\if{html}{\out{
}} +} +} +} diff --git a/man/mlr_tuners_random_search.Rd b/man/mlr_tuners_random_search.Rd index 9c82071fe..a7cf5372b 100644 --- a/man/mlr_tuners_random_search.Rd +++ b/man/mlr_tuners_random_search.Rd @@ -16,7 +16,8 @@ of termination criteria. } \section{Dictionary}{ -This \link{Tuner} can be instantiated via the \link[mlr3misc:Dictionary]{dictionary} \link{mlr_tuners} or with the associated sugar function \code{\link[=tnr]{tnr()}}:\preformatted{mlr_tuners$get("random_search") +This \link{Tuner} can be instantiated via the \link[mlr3misc:Dictionary]{dictionary} \link{mlr_tuners} or with the associated sugar function \code{\link[=tnr]{tnr()}}:\preformatted{TunerRandomSearch$new() +mlr_tuners$get("random_search") tnr("random_search") } } @@ -38,6 +39,7 @@ Other Tuner: \code{\link{mlr_tuners_design_points}}, \code{\link{mlr_tuners_gensa}}, \code{\link{mlr_tuners_grid_search}}, +\code{\link{mlr_tuners_irace}}, \code{\link{mlr_tuners}} } \concept{Tuner} diff --git a/tests/testthat/test_TunerIrace.R b/tests/testthat/test_TunerIrace.R new file mode 100644 index 000000000..8583fd0df --- /dev/null +++ b/tests/testthat/test_TunerIrace.R @@ -0,0 +1,36 @@ +context("TunerIrace") + +skip_if_not_installed("irace") + +test_that("TunerIrace", { + test_tuner("irace", term_evals = 38) +}) + +test_that("TunerIrace with int params and trafo, clock terminator", { + ps = ParamSet$new(params = list( + ParamDbl$new("cp", lower = 0.001, upper = 0.1), + ParamDbl$new("minsplit", lower = 1, upper = 10) + )) + ps$trafo = function(x, param_set) { + x$minsplit = as.integer(round(x$minsplit)) + return(x) + } + te = term("clock_time", secs = 5) + inst = TuningInstance$new(tsk("iris"), lrn("classif.rpart"), rsmp("holdout"), msr("classif.ce"), ps, te) + tt = tnr("irace") + tt$tune(inst) + d = inst$archive(unnest = "params") + expect_integer(d$minsplit) +}) + +test_that("minimize time",{ + ps = ParamSet$new(params = list( + ParamDbl$new("cp", lower = 0.001, upper = 0.1), + ParamInt$new("minsplit", lower = 1, upper = 10) + )) + te = term("clock_time", secs = 5) + inst = TuningInstance$new(tsk("iris"), lrn("classif.rpart"), rsmp("holdout"), msr("classif.ce"), ps, te) + tt = tnr("irace", capping = 1, boundMax = 1, cappingType = "best", boundType= "instance") + tt$tune(inst) + expect_double(inst$archive(unnest = "params")$minsplit) +}) From a08afbec1137d1d4bd91a15898ae84be1ef6128e Mon Sep 17 00:00:00 2001 From: Raphael Date: Mon, 9 Mar 2020 20:43:57 +0000 Subject: [PATCH 02/40] added reference --- R/TunerIrace.R | 5 +++++ man/mlr_tuners_irace.Rd | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/R/TunerIrace.R b/R/TunerIrace.R index e831f720f..74f312d7f 100644 --- a/R/TunerIrace.R +++ b/R/TunerIrace.R @@ -37,6 +37,11 @@ #' @templateVar id irace #' @template section_dictionary_tuners #' +#' @references +#' Birattari, M., Stützle, T., Paquete, L., & Varrentrapp, K. (2002). +#' A Racing Algorithm for Configuring Metaheuristics. +#' In Gecco (Vol. 2). +#' #' @family Tuner #' @export #' @examples diff --git a/man/mlr_tuners_irace.Rd b/man/mlr_tuners_irace.Rd index 52f74c3dd..1b6b0b1ee 100644 --- a/man/mlr_tuners_irace.Rd +++ b/man/mlr_tuners_irace.Rd @@ -50,6 +50,11 @@ tnr("irace") \examples{ # see ?Tuner } +\references{ +Birattari, M., Stützle, T., Paquete, L., & Varrentrapp, K. (2002). +A Racing Algorithm for Configuring Metaheuristics. +In Gecco (Vol. 2). +} \seealso{ Other Tuner: \code{\link{Tuner}}, From a40c7a4f801545102a7a943a8ddf190e57ad3722 Mon Sep 17 00:00:00 2001 From: Raphael Date: Mon, 9 Mar 2020 22:05:30 +0000 Subject: [PATCH 03/40] formatting, merging --- R/TunerIrace.R | 47 +++++++++++++++++++++++------------------------ R/irace_helpers.R | 36 ++++++++++++++++-------------------- 2 files changed, 39 insertions(+), 44 deletions(-) diff --git a/R/TunerIrace.R b/R/TunerIrace.R index 74f312d7f..6ec7d277a 100644 --- a/R/TunerIrace.R +++ b/R/TunerIrace.R @@ -52,27 +52,27 @@ TunerIrace = R6Class("TunerIrace", inherit = Tuner, #' Creates a new instance of this [R6][R6::R6Class] class. initialize = function() { ps = ParamSet$new(list( - ParamLgl$new("show.irace.output", default = FALSE), - ParamInt$new("debugLevel", default = 0, lower = 0), - ParamInt$new("seed"), - ParamDbl$new("postselection",default=0,lower=0,upper=1), - ParamInt$new("elitist",default=1,lower=0,upper=1), - ParamInt$new("elitistLimit",default=2,lower=0), - ParamInt$new("mu",default=5,lower=1), - ParamInt$new("softRestart",default=1,lower=0,upper=1), - ParamDbl$new("softRestartThreshold"), - ParamInt$new("digits", default = 4, lower = 0), - ParamFct$new("testType", default = "F-test", levels = c("F-test","t-test","t-test-bonferroni","t-test-holm")), - ParamInt$new("firstTest", default = 5, lower = 0), - ParamInt$new("eachTest",default = 1, lower = 1), - ParamDbl$new("confidence",default=0.95,lower=0,upper=1), - ParamInt$new("capping",default = 0,lower=0,upper=1), - ParamFct$new("cappingType",default="median",levels=c("median","mean","best","worst")), - ParamFct$new("boundType",default="candidate",levels=c("candidate","instance")), - ParamDbl$new("boundMax",default=0), - ParamInt$new("boundDigits",default=0), - ParamDbl$new("boundPar",default=1), - ParamDbl$new("boundAsTimeout",default=1) + ParamLgl$new("show.irace.output", default = FALSE, tags = c("meta","required")), + ParamInt$new("debugLevel", default = 0, lower = 0, tags = "irace"), + ParamInt$new("seed", tags = "irace"), + ParamDbl$new("postselection",default=0,lower=0,upper=1, tags = "irace"), + ParamInt$new("elitist",default=1,lower=0,upper=1, tags = "irace"), + ParamInt$new("elitistLimit",default=2,lower=0, tags = "irace"), + ParamInt$new("mu",default=5,lower=1, tags = "irace"), + ParamInt$new("softRestart",default=1,lower=0,upper=1, tags = "irace"), + ParamDbl$new("softRestartThreshold", tags = "irace"), + ParamInt$new("digits", default = 4, lower = 0, tags = "irace"), + ParamFct$new("testType", default = "F-test", levels = c("F-test","t-test","t-test-bonferroni","t-test-holm"), tags = "irace"), + ParamInt$new("firstTest", default = 5, lower = 0, tags = "irace"), + ParamInt$new("eachTest",default = 1, lower = 1, tags = "irace"), + ParamDbl$new("confidence",default=0.95,lower=0,upper=1, tags = "irace"), + ParamInt$new("capping",default = 0,lower=0,upper=1, tags = "irace"), + ParamFct$new("cappingType",default="median",levels=c("median","mean","best","worst"), tags = "irace"), + ParamFct$new("boundType",default="candidate",levels=c("candidate","instance"), tags = "irace"), + ParamDbl$new("boundMax",default=0, tags = "irace"), + ParamInt$new("boundDigits",default=0, tags = "irace"), + ParamDbl$new("boundPar",default=1, tags = "irace"), + ParamDbl$new("boundAsTimeout",default=1, tags = "irace") )) ps$values$show.irace.output = FALSE @@ -88,9 +88,8 @@ TunerIrace = R6Class("TunerIrace", inherit = Tuner, private = list( .tune = function(instance) { - ps = self$param_set$values[!(names(self$param_set$values) %in% c("show.irace.output"))] - g = if (self$param_set$values$show.irace.output) identity else capture.output # copied from mlr - g(irace::irace(scenario = c(makeScenario(instance), ps), + g = if (self$param_set$get_values(tag = "meta")$show.irace.output) identity else capture.output + g(irace::irace(scenario = c(makeScenario(instance), self$param_set$get_values("irace")), parameters = paradoxToIrace(instance$param_set))) } ) diff --git a/R/irace_helpers.R b/R/irace_helpers.R index ca8c837ee..7beda37d3 100644 --- a/R/irace_helpers.R +++ b/R/irace_helpers.R @@ -1,10 +1,10 @@ paradoxToIrace = function(ps){ assertClass(ps, "ParamSet") # what about ParamUty = vector numeric/real - class_lookup = data.frame(paradox = c("ParamLgl","ParamInt","ParamDbl","ParamFct"), + class_lookup = data.table(paradox = c("ParamLgl","ParamInt","ParamDbl","ParamFct"), irace = c("c","i","r","c"), stringsAsFactors = FALSE) - name <- ps$ids() - type <- class_lookup[match(ps$class, class_lookup$paradox), 2] + + type <- as.character(subset(merge(class_lookup, data.table(paradox = ps$class)), select = "irace")) range <- getIraceRange(ps) if(ps$has_deps){ condition <- getIraceCondition(ps) @@ -12,33 +12,33 @@ paradoxToIrace = function(ps){ condition = NULL } - # does mlr3tuning deal with trafo internally or passed to irace? irace only handle log - par.tab <- paste(name, '""', type, range, condition, collapse = "\n") + par.tab <- paste(ps$ids(), '""', type, range, condition, collapse = "\n") irace::readParameters(text = par.tab) } getIraceRange = function(ps){ rng = data.table(lower = ps$lower, upper = ps$upper, lvl = ps$levels) apply(rng, 1, function(x){ - if(is.na(x[[1]])) + if (is.na(x[[1]])) { return(paste0("(",paste0(x[[3]], collapse = ","),")")) - else + } else { return(paste0("(",x[[1]],",",x[[2]],")")) + } }) } getIraceCondition = function(ps){ - cond = apply(ps$deps, 1, function(x){ - if(class(x[[3]])[1] == "CondEqual"){ + cond = rbindlist(apply(ps$deps, 1, function(x){ + if(x[[3]]$type == "equal"){ condition = paste("|",x[[2]], "==", x[[3]]$rhs) } else { condition = paste("|",x[[2]],"%in%", paste0("c(",paste0(x[[3]]$rhs, collapse = ","),")")) } - }) + data.table(id = x[[1]], cond = condition) + })) - condition = data.table(id = ps$ids(), cond = "") - ind = match(ps$deps$id, condition$id) - condition$cond[ind] = cond - condition$cond + tab = merge(data.frame(id = ps$ids()), cond, by = "id", all.x = TRUE) + tab[is.na(tab)] = "" + tab } makeScenario = function(instance){ @@ -47,12 +47,8 @@ makeScenario = function(instance){ logFile = tempfile(), instances = list(instance), debugLevel = 0, - maxExperiments = ifelse(class(instance$terminator)[1] == "TerminatorEvals", - instance$terminator$param_set$values$n_evals, - 0), - maxTime = ifelse(class(instance$terminator)[1] == "TerminatorClockTime", - instance$terminator$param_set$values$secs, - 0) + maxExperiments = if(class(instance$terminator)[1] == "TerminatorEvals") instance$terminator$param_set$values$n_evals else 0, + maxTime = if(class(instance$terminator)[1] == "TerminatorClockTime") instance$terminator$param_set$values$secs else 0 ) } From 79b7cdb0f6cfdaede4d6ec95d553f4d6c7b797cb Mon Sep 17 00:00:00 2001 From: Raphael Date: Tue, 10 Mar 2020 08:11:50 +0000 Subject: [PATCH 04/40] removed parameter tags, fixed paradox2irace --- R/TunerIrace.R | 49 ++++++++++++++++++++++++----------------------- R/irace_helpers.R | 2 +- 2 files changed, 26 insertions(+), 25 deletions(-) diff --git a/R/TunerIrace.R b/R/TunerIrace.R index 6ec7d277a..b75d4bd5d 100644 --- a/R/TunerIrace.R +++ b/R/TunerIrace.R @@ -52,27 +52,27 @@ TunerIrace = R6Class("TunerIrace", inherit = Tuner, #' Creates a new instance of this [R6][R6::R6Class] class. initialize = function() { ps = ParamSet$new(list( - ParamLgl$new("show.irace.output", default = FALSE, tags = c("meta","required")), - ParamInt$new("debugLevel", default = 0, lower = 0, tags = "irace"), - ParamInt$new("seed", tags = "irace"), - ParamDbl$new("postselection",default=0,lower=0,upper=1, tags = "irace"), - ParamInt$new("elitist",default=1,lower=0,upper=1, tags = "irace"), - ParamInt$new("elitistLimit",default=2,lower=0, tags = "irace"), - ParamInt$new("mu",default=5,lower=1, tags = "irace"), - ParamInt$new("softRestart",default=1,lower=0,upper=1, tags = "irace"), - ParamDbl$new("softRestartThreshold", tags = "irace"), - ParamInt$new("digits", default = 4, lower = 0, tags = "irace"), - ParamFct$new("testType", default = "F-test", levels = c("F-test","t-test","t-test-bonferroni","t-test-holm"), tags = "irace"), - ParamInt$new("firstTest", default = 5, lower = 0, tags = "irace"), - ParamInt$new("eachTest",default = 1, lower = 1, tags = "irace"), - ParamDbl$new("confidence",default=0.95,lower=0,upper=1, tags = "irace"), - ParamInt$new("capping",default = 0,lower=0,upper=1, tags = "irace"), - ParamFct$new("cappingType",default="median",levels=c("median","mean","best","worst"), tags = "irace"), - ParamFct$new("boundType",default="candidate",levels=c("candidate","instance"), tags = "irace"), - ParamDbl$new("boundMax",default=0, tags = "irace"), - ParamInt$new("boundDigits",default=0, tags = "irace"), - ParamDbl$new("boundPar",default=1, tags = "irace"), - ParamDbl$new("boundAsTimeout",default=1, tags = "irace") + ParamLgl$new("show.irace.output", default = FALSE), + ParamInt$new("debugLevel", default = 0, lower = 0), + ParamInt$new("seed"), + ParamDbl$new("postselection",default=0,lower=0,upper=1), + ParamInt$new("elitist",default=1,lower=0,upper=1), + ParamInt$new("elitistLimit",default=2,lower=0), + ParamInt$new("mu",default=5,lower=1), + ParamInt$new("softRestart",default=1,lower=0,upper=1), + ParamDbl$new("softRestartThreshold"), + ParamInt$new("digits", default = 4, lower = 0), + ParamFct$new("testType", default = "F-test", levels = c("F-test","t-test","t-test-bonferroni","t-test-holm")), + ParamInt$new("firstTest", default = 5, lower = 0), + ParamInt$new("eachTest",default = 1, lower = 1), + ParamDbl$new("confidence",default=0.95,lower=0,upper=1), + ParamInt$new("capping",default = 0,lower=0,upper=1), + ParamFct$new("cappingType",default="median",levels=c("median","mean","best","worst")), + ParamFct$new("boundType",default="candidate",levels=c("candidate","instance")), + ParamDbl$new("boundMax",default=0), + ParamInt$new("boundDigits",default=0), + ParamDbl$new("boundPar",default=1), + ParamDbl$new("boundAsTimeout",default=1) )) ps$values$show.irace.output = FALSE @@ -80,7 +80,7 @@ TunerIrace = R6Class("TunerIrace", inherit = Tuner, super$initialize( param_set = ps, param_classes = c("ParamDbl","ParamInt","ParamFct","ParamLgl"), - properties = character(), + properties = "dependencies", packages = "irace" ) } @@ -88,8 +88,9 @@ TunerIrace = R6Class("TunerIrace", inherit = Tuner, private = list( .tune = function(instance) { - g = if (self$param_set$get_values(tag = "meta")$show.irace.output) identity else capture.output - g(irace::irace(scenario = c(makeScenario(instance), self$param_set$get_values("irace")), + g = if (self$param_set$values$show.irace.output) identity else capture.output + g(irace::irace(scenario = c(makeScenario(instance), + self$param_set$values[names(self$param_set$values) %nin% "show.irace.output"]), parameters = paradoxToIrace(instance$param_set))) } ) diff --git a/R/irace_helpers.R b/R/irace_helpers.R index 7beda37d3..ff11509ac 100644 --- a/R/irace_helpers.R +++ b/R/irace_helpers.R @@ -4,7 +4,7 @@ paradoxToIrace = function(ps){ class_lookup = data.table(paradox = c("ParamLgl","ParamInt","ParamDbl","ParamFct"), irace = c("c","i","r","c"), stringsAsFactors = FALSE) - type <- as.character(subset(merge(class_lookup, data.table(paradox = ps$class)), select = "irace")) + type <- unlist(subset(merge(class_lookup, data.table(paradox = ps$class)), select = "irace")) range <- getIraceRange(ps) if(ps$has_deps){ condition <- getIraceCondition(ps) From e8659dbb94c5d3998b6eb106a863ee8f6d24c008 Mon Sep 17 00:00:00 2001 From: Raphael Date: Tue, 10 Mar 2020 08:18:59 +0000 Subject: [PATCH 05/40] camel case -> underscore, arrows -> equals, added spaces to ifs `targetRunner` kept as-is to be in line with `irace` convention --- R/TunerIrace.R | 4 ++-- R/irace_helpers.R | 34 ++++++++++++++++++---------------- 2 files changed, 20 insertions(+), 18 deletions(-) diff --git a/R/TunerIrace.R b/R/TunerIrace.R index b75d4bd5d..14d103a4b 100644 --- a/R/TunerIrace.R +++ b/R/TunerIrace.R @@ -89,9 +89,9 @@ TunerIrace = R6Class("TunerIrace", inherit = Tuner, private = list( .tune = function(instance) { g = if (self$param_set$values$show.irace.output) identity else capture.output - g(irace::irace(scenario = c(makeScenario(instance), + g(irace::irace(scenario = c(make_scenario(instance), self$param_set$values[names(self$param_set$values) %nin% "show.irace.output"]), - parameters = paradoxToIrace(instance$param_set))) + parameters = paradox_to_irace(instance$param_set))) } ) ) diff --git a/R/irace_helpers.R b/R/irace_helpers.R index ff11509ac..9603cdefd 100644 --- a/R/irace_helpers.R +++ b/R/irace_helpers.R @@ -1,21 +1,22 @@ -paradoxToIrace = function(ps){ +paradox_to_irace = function(ps){ assertClass(ps, "ParamSet") # what about ParamUty = vector numeric/real class_lookup = data.table(paradox = c("ParamLgl","ParamInt","ParamDbl","ParamFct"), irace = c("c","i","r","c"), stringsAsFactors = FALSE) - type <- unlist(subset(merge(class_lookup, data.table(paradox = ps$class)), select = "irace")) - range <- getIraceRange(ps) - if(ps$has_deps){ - condition <- getIraceCondition(ps) + type = unlist(subset(merge(class_lookup, data.table(paradox = ps$class)), select = "irace")) + range = get_irace_range(ps) + if (ps$has_deps) { + condition = get_irace_condition(ps) } else { condition = NULL } - par.tab <- paste(ps$ids(), '""', type, range, condition, collapse = "\n") - irace::readParameters(text = par.tab) + par.tab = paste(ps$ids(), '""', type, range, condition, collapse = "\n") + + return(irace::readParameters(text = par.tab)) } -getIraceRange = function(ps){ +get_irace_range = function(ps){ rng = data.table(lower = ps$lower, upper = ps$upper, lvl = ps$levels) apply(rng, 1, function(x){ @@ -26,9 +27,9 @@ getIraceRange = function(ps){ } }) } -getIraceCondition = function(ps){ +get_irace_condition = function(ps){ cond = rbindlist(apply(ps$deps, 1, function(x){ - if(x[[3]]$type == "equal"){ + if (x[[3]]$type == "equal") { condition = paste("|",x[[2]], "==", x[[3]]$rhs) } else { condition = paste("|",x[[2]],"%in%", paste0("c(",paste0(x[[3]]$rhs, collapse = ","),")")) @@ -38,17 +39,17 @@ getIraceCondition = function(ps){ tab = merge(data.frame(id = ps$ids()), cond, by = "id", all.x = TRUE) tab[is.na(tab)] = "" - tab -} -makeScenario = function(instance){ + return(tab) +} +make_scenario = function(instance){ list( targetRunner = targetRunner, logFile = tempfile(), instances = list(instance), debugLevel = 0, - maxExperiments = if(class(instance$terminator)[1] == "TerminatorEvals") instance$terminator$param_set$values$n_evals else 0, - maxTime = if(class(instance$terminator)[1] == "TerminatorClockTime") instance$terminator$param_set$values$secs else 0 + maxExperiments = if (class(instance$terminator)[1] == "TerminatorEvals") instance$terminator$param_set$values$n_evals else 0, + maxTime = if (class(instance$terminator)[1] == "TerminatorClockTime") instance$terminator$param_set$values$secs else 0 ) } @@ -56,5 +57,6 @@ targetRunner = function(experiment, scenario){ t0 = Sys.time() cost = scenario$instances[[1]]$eval_batch(as.data.table(experiment$configuration))$perf t1 = Sys.time() - list(cost = cost, time = as.numeric(t1-t0)) + + return(list(cost = cost, time = as.numeric(t1-t0))) } From c12a293c4bd28b51c22daa8fa457a85bd10cc8a0 Mon Sep 17 00:00:00 2001 From: Raphael Date: Tue, 10 Mar 2020 10:27:01 +0000 Subject: [PATCH 06/40] tests, params, fixes --- DESCRIPTION | 3 + R/TunerIrace.R | 10 +- R/irace_helpers.R | 25 ++++- man/mlr_tuners_irace.Rd | 4 + tests/testthat/helper.R | 25 +++++ tests/testthat/test_TunerIrace.R | 182 ++++++++++++++++++++++++++++++- 6 files changed, 239 insertions(+), 10 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index 851f6cadd..df984bd77 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -42,8 +42,11 @@ Imports: paradox, R6 Suggests: + e1071, GenSA, + glmnet, irace, + mlr3learners, mlr3pipelines, rpart, testthat diff --git a/R/TunerIrace.R b/R/TunerIrace.R index 14d103a4b..956de688b 100644 --- a/R/TunerIrace.R +++ b/R/TunerIrace.R @@ -13,6 +13,10 @@ #' * `postselection` (`numeric(1)`) #' * `elitist` (`integer(1)`) #' * `elitistLimit` (`integer(1)`) +#' * `nbIterations` (`integer(1)`) +#' * `nbExperimentsPerIteration` (`integer(1)`) +#' * `minNbSurvival` (`integer(1)`) +#' * `nbConfigurations` (`integer(1)`) #' * `mu` (`integer(1)`) #' * `softRestart` (`integer(1)`) #' * `softRestartThreshold` (`numeric(1)`) @@ -58,10 +62,14 @@ TunerIrace = R6Class("TunerIrace", inherit = Tuner, ParamDbl$new("postselection",default=0,lower=0,upper=1), ParamInt$new("elitist",default=1,lower=0,upper=1), ParamInt$new("elitistLimit",default=2,lower=0), + ParamInt$new("nbIterations",default=0,lower=0), + ParamInt$new("nbExperimentsPerIteration",default=0,lower=0), + ParamInt$new("minNbSurvival",default=0,lower=0), + ParamInt$new("nbConfigurations",default=0,lower=0), ParamInt$new("mu",default=5,lower=1), ParamInt$new("softRestart",default=1,lower=0,upper=1), ParamDbl$new("softRestartThreshold"), - ParamInt$new("digits", default = 4, lower = 0), + ParamInt$new("digits", default = 4, lower = 1, upper = 15), ParamFct$new("testType", default = "F-test", levels = c("F-test","t-test","t-test-bonferroni","t-test-holm")), ParamInt$new("firstTest", default = 5, lower = 0), ParamInt$new("eachTest",default = 1, lower = 1), diff --git a/R/irace_helpers.R b/R/irace_helpers.R index 9603cdefd..c3f62090b 100644 --- a/R/irace_helpers.R +++ b/R/irace_helpers.R @@ -4,7 +4,7 @@ paradox_to_irace = function(ps){ class_lookup = data.table(paradox = c("ParamLgl","ParamInt","ParamDbl","ParamFct"), irace = c("c","i","r","c"), stringsAsFactors = FALSE) - type = unlist(subset(merge(class_lookup, data.table(paradox = ps$class)), select = "irace")) + type = unlist(subset(merge(data.table(paradox = ps$class), class_lookup, sort = FALSE), select = "irace")) range = get_irace_range(ps) if (ps$has_deps) { condition = get_irace_condition(ps) @@ -12,7 +12,7 @@ paradox_to_irace = function(ps){ condition = NULL } - par.tab = paste(ps$ids(), '""', type, range, condition, collapse = "\n") + par.tab = paste(ps$ids(), '""', type, range, condition$cond, collapse = "\n") return(irace::readParameters(text = par.tab)) } @@ -29,15 +29,21 @@ get_irace_range = function(ps){ } get_irace_condition = function(ps){ cond = rbindlist(apply(ps$deps, 1, function(x){ + on = x[[2]] + cond = x[[3]]$rhs + if (is.character(cond)) { + cond = paste0("'", cond ,"'") + } if (x[[3]]$type == "equal") { - condition = paste("|",x[[2]], "==", x[[3]]$rhs) + condition = paste("|",x[[2]], "==", cond) } else { - condition = paste("|",x[[2]],"%in%", paste0("c(",paste0(x[[3]]$rhs, collapse = ","),")")) + condition = paste("|",x[[2]],"%in%", paste0("c(",paste0(cond, collapse = ","),")")) } data.table(id = x[[1]], cond = condition) })) - tab = merge(data.frame(id = ps$ids()), cond, by = "id", all.x = TRUE) + # coercion back and forth from frame/table is due to data.frame sorting even when sort = FALSE + tab = data.frame(merge(data.table(id = ps$ids()), cond, by = "id", all.x = TRUE, sort = FALSE)) tab[is.na(tab)] = "" return(tab) @@ -55,7 +61,14 @@ make_scenario = function(instance){ targetRunner = function(experiment, scenario){ t0 = Sys.time() - cost = scenario$instances[[1]]$eval_batch(as.data.table(experiment$configuration))$perf + config = as.data.table(lapply(experiment$configuration, function(x){ + if (x %in% c("TRUE", "FALSE")) { + return(as.logical(x)) + } else { + return(x) + } + })) + cost = scenario$instances[[1]]$eval_batch(config)$perf t1 = Sys.time() return(list(cost = cost, time = as.numeric(t1-t0))) diff --git a/man/mlr_tuners_irace.Rd b/man/mlr_tuners_irace.Rd index 1b6b0b1ee..566069b18 100644 --- a/man/mlr_tuners_irace.Rd +++ b/man/mlr_tuners_irace.Rd @@ -16,6 +16,10 @@ Subclass for iterated racing tuning calling \code{\link[irace:irace]{irace::irac \item \code{postselection} (\code{numeric(1)}) \item \code{elitist} (\code{integer(1)}) \item \code{elitistLimit} (\code{integer(1)}) +\item \code{nbIterations} (\code{integer(1)}) +\item \code{nbExperimentsPerIteration} (\code{integer(1)}) +\item \code{minNbSurvival} (\code{integer(1)}) +\item \code{nbConfigurations} (\code{integer(1)}) \item \code{mu} (\code{integer(1)}) \item \code{softRestart} (\code{integer(1)}) \item \code{softRestartThreshold} (\code{numeric(1)}) diff --git a/tests/testthat/helper.R b/tests/testthat/helper.R index 1af30c933..652986da0 100644 --- a/tests/testthat/helper.R +++ b/tests/testthat/helper.R @@ -198,3 +198,28 @@ LearnerRegrDepParams = R6Class("LearnerRegrDepParams", inherit = LearnerRegr, ) # mlr3::mlr_learners$add("regr.depparams", LearnerRegrDepParams) +expect_irace_parameters = function(parameters, names, types, domain, conditions, hierarchy){ + expect_list(parameters, len = 11, any.missing = FALSE) + expect_equal(names(parameters), c("names", "types", "switches", "domain", "conditions","isFixed", + "transform","hierarchy","nbParameters","nbFixed","nbVariable")) + expect_equal(parameters$names, names) + expect_equal(parameters$types, set_names(types, names)) + expect_equal(parameters$switches, named_vector(names, "")) + expect_equal(parameters$domain, domain) + if (missing(conditions)) { + expect_equal(parameters$conditions, named_list(names, TRUE)) + } else { + # can't compare expressions directly + expect_equal(as.character(parameters$conditions), as.character(conditions)) + } + expect_equal(parameters$isFixed, named_vector(names, FALSE)) + expect_equal(parameters$transform, named_list(names, "")) + if (missing(hierarchy)) { + expect_equal(parameters$hierarchy, named_vector(names, 1)) + } else { + expect_equal(parameters$hierarchy, set_names(hierarchy, names)) + } + expect_equal(parameters$nbParameters, length(names)) + expect_equal(parameters$nbFixed, 0) + expect_equal(parameters$nbVariable, length(names)) +} diff --git a/tests/testthat/test_TunerIrace.R b/tests/testthat/test_TunerIrace.R index 8583fd0df..e892b2b79 100644 --- a/tests/testthat/test_TunerIrace.R +++ b/tests/testthat/test_TunerIrace.R @@ -1,4 +1,6 @@ context("TunerIrace") +library(mlr3learners) +set.seed(1) skip_if_not_installed("irace") @@ -15,22 +17,196 @@ test_that("TunerIrace with int params and trafo, clock terminator", { x$minsplit = as.integer(round(x$minsplit)) return(x) } - te = term("clock_time", secs = 5) - inst = TuningInstance$new(tsk("iris"), lrn("classif.rpart"), rsmp("holdout"), msr("classif.ce"), ps, te) + inst = TuningInstance$new(tsk("iris"), lrn("classif.rpart"), rsmp("holdout"), msr("classif.ce"), + ps, term("clock_time", secs = 10)) tt = tnr("irace") tt$tune(inst) d = inst$archive(unnest = "params") expect_integer(d$minsplit) }) +test_that("TunerIrace with dependencies",{ + ps = ParamSet$new(params = list( + ParamDbl$new("cost", lower = 0, upper = 1), + ParamFct$new("kernel", levels = c("linear","polynomial")) + )) + ps$add_dep("cost","kernel",CondEqual$new("polynomial")) + inst = TuningInstance$new(tsk("iris"), lrn("classif.svm",type="C-classification"), + rsmp("holdout"), msr("classif.ce"), ps, term("clock_time", secs = 8)) + tt = tnr("irace") + # should it possible to return kernel = linear...? + expect_silent(tt$tune(inst)) +}) + test_that("minimize time",{ ps = ParamSet$new(params = list( ParamDbl$new("cp", lower = 0.001, upper = 0.1), ParamInt$new("minsplit", lower = 1, upper = 10) )) - te = term("clock_time", secs = 5) + te = term("clock_time", secs = 11) inst = TuningInstance$new(tsk("iris"), lrn("classif.rpart"), rsmp("holdout"), msr("classif.ce"), ps, te) tt = tnr("irace", capping = 1, boundMax = 1, cappingType = "best", boundType= "instance") tt$tune(inst) expect_double(inst$archive(unnest = "params")$minsplit) }) + +test_that("paradox_to_irace no dependencies",{ + ps = ParamSet$new(params = list( + ParamLgl$new("lgl") + )) + expect_irace_parameters(parameters = paradox_to_irace(ps), names = "lgl", types = "c", + domain = list(lgl = c("TRUE", "FALSE")), conditions = list(lgl = TRUE)) + + ps = ParamSet$new(params = list( + ParamUty$new("uty") + )) + expect_error(paradox_to_irace(ps)) + + ps = ParamSet$new(params = list( + ParamDbl$new("dbl", lower = 0.1, upper = 0.3), + ParamInt$new("int", lower = 1, upper = 9), + ParamFct$new("fct", levels = c("a","b","c")), + ParamLgl$new("lgl") + )) + expect_irace_parameters(parameters = paradox_to_irace(ps), + names = c("dbl","int","fct","lgl"), + types = c("r","i","c","c"), + domain = list(dbl = c(0.1,0.3), int = c(1,9), fct = c("a","b","c"), + lgl = c("TRUE", "FALSE"))) + + # double checking previous bug in merge sort + ps = ParamSet$new(params = list( + ParamFct$new("fct", levels = c("a","b","c")), + ParamInt$new("int1", lower = 1, upper = 9), + ParamDbl$new("dbl", lower = 0.1, upper = 0.3), + ParamInt$new("int2", lower = 10, upper = 90), + ParamLgl$new("lgl") + )) + expect_irace_parameters(parameters = paradox_to_irace(ps), + names = c("fct","int1","dbl","int2","lgl"), + types = c("c","i","r","i","c"), + domain = list(fct = c("a","b","c"), int1 = c(1,9), dbl = c(0.1,0.3), + int2 = c(10,90), lgl = c("TRUE", "FALSE"))) +}) + +test_that("paradox_to_irace dependencies",{ + ps = ParamSet$new(params = list( + ParamLgl$new("a"), + ParamInt$new("b", lower = 1, upper = 9)) + ) + ps$add_dep("b", "a", CondEqual$new(TRUE)) + expect_irace_parameters(parameters = paradox_to_irace(ps), names = c("a","b"), types = c("c","i"), + domain = list(a = c("TRUE", "FALSE"), b = c(1,9)), + conditions = list(a = TRUE, b = expression(a == TRUE)), + hierarchy = c(1,2)) + + ps = ParamSet$new(params = list( + ParamLgl$new("a"), + ParamFct$new("c", levels = c("lvl1","lvl2")), + ParamInt$new("b", lower = 1, upper = 9) + )) + ps$add_dep("b", "a", CondEqual$new(TRUE)) + ps$add_dep("c", "b", CondAnyOf$new(c(2,5,7))) + expect_irace_parameters(parameters = paradox_to_irace(ps), names = c("a","c","b"), + types = c("c","c","i"), + domain = list(a = c("TRUE", "FALSE"), c = c("lvl1","lvl2"), b = c(1,9)), + conditions = list(a = TRUE, b = expression(a == TRUE), c = expression(b %in% c(2,5,7))), + hierarchy = c(1,3,2)) + + + ps = ParamSet$new(params = list( + ParamLgl$new("a"), + ParamInt$new("b", lower = 1, upper = 9), + ParamFct$new("c", levels = c("lvl1","lvl2")), + ParamDbl$new("d", lower = 0, upper = 1) + )) + ps$add_dep("b", "a", CondEqual$new(TRUE)) + ps$add_dep("a", "c", CondEqual$new("lvl1")) + ps$add_dep("d", "c", CondAnyOf$new(c("lvl1","lvl2"))) + expect_irace_parameters(parameters = paradox_to_irace(ps), names = c("a","b","c","d"), + types = c("c","i","c","r"), + domain = list(a = c("TRUE", "FALSE"), b = c(1,9), c = c("lvl1","lvl2"), d = c(0,1)), + conditions = list(c = TRUE, a = expression(c == "lvl1"), + d = expression(c %in% c('lvl1', 'lvl2')),b = expression(a == TRUE)), + hierarchy = c(2,3,1,2)) +}) + +test_that("TunerIrace works with logical params", { + ps = ParamSet$new(params = list( + ParamLgl$new("intercept") + )) + inst = TuningInstance$new(tsk("mtcars"), lrn("regr.glmnet"), rsmp("holdout"), msr("regr.mse"), + ps, term("clock_time", secs = 10)) + tt = tnr("irace") + expect_warning(tt$tune(inst)) + d = inst$archive(unnest = "params") + expect_logical(d$intercept) +}) + +test_that("TunerIrace works with tune.threshold", { + ps = ParamSet$new(params = list( + ParamInt$new("minsplit", lower = 1, upper = 3) + )) + te = term("evals", n_evals = 50) + tt = tnr("irace", nbIterations = 1L, minNbSurvival = 1) + inst = TuningInstance$new(tsk("iris"), lrn("classif.rpart"), rsmp("holdout", ratio = 0.1), msr("classif.ce"), ps, te) + expect_silent(tt$tune(inst)) +}) + +test_that("TunerIrace uses digits", { + te = term("evals", n_evals = 30) + tt = tnr("irace", nbIterations = 1L, minNbSurvival = 1) + ps = ParamSet$new(params = list( + ParamDbl$new("cost", lower = pi * 1e-20,upper = 5.242e12) + )) + inst = TuningInstance$new(tsk("iris"), lrn("classif.svm", type="C-classification"), rsmp("holdout"), msr("classif.ce"), ps, te) + expect_silent(tt$tune(inst)) + + + ps = ParamSet$new(params = list( + ParamDbl$new("cp", lower = 1e-5, upper = 1e-4) + )) + inst = TuningInstance$new(tsk("iris"), lrn("classif.rpart"), rsmp("holdout"), msr("classif.ce"), ps, te) + tt = tnr("irace", nbIterations = 1L, digits = 5L) + expect_error(tt$tune(inst)) + + te = term("evals", n_evals = 40) + tt = tnr("irace", digits = 3L) + inst = TuningInstance$new(tsk("iris"), lrn("classif.rpart"), rsmp("holdout"), msr("classif.ce"), ps, te) + expect_error(tt$tune(inst)) +}) + +test_that("Error in hyperparameter tuning with scientific notation for lower/upper boundaries", { + ps = ParamSet$new(params = list( + ParamDbl$new("cp", lower = 1e-5, upper = 1e-4) + )) + inst = TuningInstance$new(tsk("iris"), lrn("classif.rpart"), rsmp("holdout"), msr("classif.ce"), ps, term("evals", n_evals = 30)) + tt = tnr("irace", nbIterations = 1L, digits = 6L) + expect_silent(tt$tune(inst)) +}) + +# we had a bug here, see (mlr) issue #627 +test_that("irace works with unnamed discrete values", { + ps = ParamSet$new(params = list( + ParamInt$new("minsplit", lower = 2L, upper = 7L) + )) + inst = TuningInstance$new(tsk("iris"), lrn("classif.rpart"), rsmp("holdout"), msr("classif.ce"), + ps, term("evals", n_evals = 50)) + tt = tnr("irace") + expect_silent(tt$tune(inst)) +}) + +# shouldn't paradox check this? +test_that("irace handles parameters with unsatisfiable requirement gracefully", { + skip_on_os("windows") + tt = tnr("irace", nbIterations = 1L, minNbSurvival = 1L) + + ps = ParamSet$new(params = list( + ParamDbl$new("cost", lower = 0, upper = 1), + ParamFct$new("kernel", levels = c("linear","polynomial")) + )) + ps$add_dep("kernel","cost",CondEqual$new(-1)) # cost never feasible + inst = TuningInstance$new(tsk("iris"), lrn("classif.svm", type= "C-classification"), rsmp("holdout"), msr("classif.ce"), + ps, term("evals", n_evals = 50)) + expect_warning(tt$tune(inst)) +}) From bdf853c5cae4df329facfcba90e239c740ce8040 Mon Sep 17 00:00:00 2001 From: Raphael Date: Tue, 10 Mar 2020 13:26:56 +0000 Subject: [PATCH 07/40] Update test_TunerIrace.R --- tests/testthat/test_TunerIrace.R | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/tests/testthat/test_TunerIrace.R b/tests/testthat/test_TunerIrace.R index e892b2b79..f034726a5 100644 --- a/tests/testthat/test_TunerIrace.R +++ b/tests/testthat/test_TunerIrace.R @@ -166,10 +166,6 @@ test_that("TunerIrace uses digits", { ps = ParamSet$new(params = list( ParamDbl$new("cp", lower = 1e-5, upper = 1e-4) )) - inst = TuningInstance$new(tsk("iris"), lrn("classif.rpart"), rsmp("holdout"), msr("classif.ce"), ps, te) - tt = tnr("irace", nbIterations = 1L, digits = 5L) - expect_error(tt$tune(inst)) - te = term("evals", n_evals = 40) tt = tnr("irace", digits = 3L) inst = TuningInstance$new(tsk("iris"), lrn("classif.rpart"), rsmp("holdout"), msr("classif.ce"), ps, te) @@ -208,5 +204,5 @@ test_that("irace handles parameters with unsatisfiable requirement gracefully", ps$add_dep("kernel","cost",CondEqual$new(-1)) # cost never feasible inst = TuningInstance$new(tsk("iris"), lrn("classif.svm", type= "C-classification"), rsmp("holdout"), msr("classif.ce"), ps, term("evals", n_evals = 50)) - expect_warning(tt$tune(inst)) + expect_silent(suppressWarnings(tt$tune(inst))) }) From ff648383844c4cc04a7098d1c70211ce8dd8bde8 Mon Sep 17 00:00:00 2001 From: Raphael Date: Tue, 10 Mar 2020 13:47:40 +0000 Subject: [PATCH 08/40] Update test_TunerIrace.R --- tests/testthat/test_TunerIrace.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/testthat/test_TunerIrace.R b/tests/testthat/test_TunerIrace.R index f034726a5..5a5e1562f 100644 --- a/tests/testthat/test_TunerIrace.R +++ b/tests/testthat/test_TunerIrace.R @@ -189,7 +189,7 @@ test_that("irace works with unnamed discrete values", { inst = TuningInstance$new(tsk("iris"), lrn("classif.rpart"), rsmp("holdout"), msr("classif.ce"), ps, term("evals", n_evals = 50)) tt = tnr("irace") - expect_silent(tt$tune(inst)) + expect_silent(suppressWarnings(tt$tune(inst))) }) # shouldn't paradox check this? From b2adc83252a148d8f9b93552c7c5d8cc684f6f5f Mon Sep 17 00:00:00 2001 From: Raphael Date: Tue, 10 Mar 2020 13:57:43 +0000 Subject: [PATCH 09/40] Update test_TunerIrace.R --- tests/testthat/test_TunerIrace.R | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/testthat/test_TunerIrace.R b/tests/testthat/test_TunerIrace.R index 5a5e1562f..7fe61bf6f 100644 --- a/tests/testthat/test_TunerIrace.R +++ b/tests/testthat/test_TunerIrace.R @@ -35,7 +35,7 @@ test_that("TunerIrace with dependencies",{ rsmp("holdout"), msr("classif.ce"), ps, term("clock_time", secs = 8)) tt = tnr("irace") # should it possible to return kernel = linear...? - expect_silent(tt$tune(inst)) + expect_silent(suppressWarnings(tt$tune(inst))) }) test_that("minimize time",{ @@ -177,7 +177,7 @@ test_that("Error in hyperparameter tuning with scientific notation for lower/upp ParamDbl$new("cp", lower = 1e-5, upper = 1e-4) )) inst = TuningInstance$new(tsk("iris"), lrn("classif.rpart"), rsmp("holdout"), msr("classif.ce"), ps, term("evals", n_evals = 30)) - tt = tnr("irace", nbIterations = 1L, digits = 6L) + tt = tnr("irace", nbIterations = 1L) expect_silent(tt$tune(inst)) }) From 09d547e8bcee0563522070451c3d0bd4668af8c3 Mon Sep 17 00:00:00 2001 From: Raphael Date: Tue, 10 Mar 2020 14:05:14 +0000 Subject: [PATCH 10/40] Update test_TunerIrace.R --- tests/testthat/test_TunerIrace.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/testthat/test_TunerIrace.R b/tests/testthat/test_TunerIrace.R index 7fe61bf6f..1798882a9 100644 --- a/tests/testthat/test_TunerIrace.R +++ b/tests/testthat/test_TunerIrace.R @@ -174,7 +174,7 @@ test_that("TunerIrace uses digits", { test_that("Error in hyperparameter tuning with scientific notation for lower/upper boundaries", { ps = ParamSet$new(params = list( - ParamDbl$new("cp", lower = 1e-5, upper = 1e-4) + ParamDbl$new("cp", lower = 1e-3, upper = 1e-1) )) inst = TuningInstance$new(tsk("iris"), lrn("classif.rpart"), rsmp("holdout"), msr("classif.ce"), ps, term("evals", n_evals = 30)) tt = tnr("irace", nbIterations = 1L) From 00e0b7e16a414fc5569a047fdda0c611dfb1095c Mon Sep 17 00:00:00 2001 From: Raphael Sonabend Date: Tue, 8 Dec 2020 22:21:01 +0000 Subject: [PATCH 11/40] update irace --- NAMESPACE | 2 +- R/TunerIrace.R | 8 +-- R/irace_helpers.R | 16 +++-- man/mlr_tuners_irace.Rd | 13 ++-- tests/testthat/helper.R | 13 +++- tests/testthat/test_TunerIrace.R | 110 ++++++++++++++----------------- 6 files changed, 82 insertions(+), 80 deletions(-) diff --git a/NAMESPACE b/NAMESPACE index f2ea8ec02..85c74fb06 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -9,8 +9,8 @@ export(TunerDesignPoints) export(TunerFromOptimizer) export(TunerGenSA) export(TunerGridSearch) -export(TunerNLoptr) export(TunerIrace) +export(TunerNLoptr) export(TunerRandomSearch) export(TuningInstanceMultiCrit) export(TuningInstanceSingleCrit) diff --git a/R/TunerIrace.R b/R/TunerIrace.R index 956de688b..11f59a6fb 100644 --- a/R/TunerIrace.R +++ b/R/TunerIrace.R @@ -88,18 +88,18 @@ TunerIrace = R6Class("TunerIrace", inherit = Tuner, super$initialize( param_set = ps, param_classes = c("ParamDbl","ParamInt","ParamFct","ParamLgl"), - properties = "dependencies", + properties = c("dependencies", "single-crit"), packages = "irace" ) } ), private = list( - .tune = function(instance) { + .optimize = function(inst) { g = if (self$param_set$values$show.irace.output) identity else capture.output - g(irace::irace(scenario = c(make_scenario(instance), + g(irace::irace(scenario = c(make_scenario(inst), self$param_set$values[names(self$param_set$values) %nin% "show.irace.output"]), - parameters = paradox_to_irace(instance$param_set))) + parameters = paradox_to_irace(inst$search_space))) } ) ) diff --git a/R/irace_helpers.R b/R/irace_helpers.R index c3f62090b..ea81b296f 100644 --- a/R/irace_helpers.R +++ b/R/irace_helpers.R @@ -4,7 +4,8 @@ paradox_to_irace = function(ps){ class_lookup = data.table(paradox = c("ParamLgl","ParamInt","ParamDbl","ParamFct"), irace = c("c","i","r","c"), stringsAsFactors = FALSE) - type = unlist(subset(merge(data.table(paradox = ps$class), class_lookup, sort = FALSE), select = "irace")) + type = unlist(subset(merge(data.table(paradox = ps$class), class_lookup, sort = FALSE), + select = "irace")) range = get_irace_range(ps) if (ps$has_deps) { condition = get_irace_condition(ps) @@ -48,19 +49,20 @@ get_irace_condition = function(ps){ return(tab) } -make_scenario = function(instance){ +make_scenario = function(inst){ list( targetRunner = targetRunner, logFile = tempfile(), - instances = list(instance), + instances = list(inst), debugLevel = 0, - maxExperiments = if (class(instance$terminator)[1] == "TerminatorEvals") instance$terminator$param_set$values$n_evals else 0, - maxTime = if (class(instance$terminator)[1] == "TerminatorClockTime") instance$terminator$param_set$values$secs else 0 + maxExperiments = if (class(inst$terminator)[1] == "TerminatorEvals") inst$terminator$param_set$values$n_evals else 0, + maxTime = if (class(inst$terminator)[1] == "TerminatorRunTime") inst$terminator$param_set$values$secs else 0 ) } targetRunner = function(experiment, scenario){ t0 = Sys.time() + # fix logicals config = as.data.table(lapply(experiment$configuration, function(x){ if (x %in% c("TRUE", "FALSE")) { return(as.logical(x)) @@ -68,8 +70,8 @@ targetRunner = function(experiment, scenario){ return(x) } })) - cost = scenario$instances[[1]]$eval_batch(config)$perf + cost = scenario$instances[[1]]$eval_batch(config)[[1]] t1 = Sys.time() - return(list(cost = cost, time = as.numeric(t1-t0))) + return(list(cost = cost, time = as.numeric(t1 - t0))) } diff --git a/man/mlr_tuners_irace.Rd b/man/mlr_tuners_irace.Rd index 566069b18..cc4aff26a 100644 --- a/man/mlr_tuners_irace.Rd +++ b/man/mlr_tuners_irace.Rd @@ -45,7 +45,8 @@ where our terminators allow to obtain the same behavior. \section{Dictionary}{ -This \link{Tuner} can be instantiated via the \link[mlr3misc:Dictionary]{dictionary} \link{mlr_tuners} or with the associated sugar function \code{\link[=tnr]{tnr()}}:\preformatted{TunerIrace$new() +This \link{Tuner} can be instantiated via the \link[mlr3misc:Dictionary]{dictionary} +\link{mlr_tuners} or with the associated sugar function \code{\link[=tnr]{tnr()}}:\preformatted{TunerIrace$new() mlr_tuners$get("irace") tnr("irace") } @@ -61,12 +62,12 @@ In Gecco (Vol. 2). } \seealso{ Other Tuner: -\code{\link{Tuner}}, +\code{\link{mlr_tuners_cmaes}}, \code{\link{mlr_tuners_design_points}}, \code{\link{mlr_tuners_gensa}}, \code{\link{mlr_tuners_grid_search}}, -\code{\link{mlr_tuners_random_search}}, -\code{\link{mlr_tuners}} +\code{\link{mlr_tuners_nloptr}}, +\code{\link{mlr_tuners_random_search}} } \concept{Tuner} \section{Super class}{ @@ -83,13 +84,14 @@ Other Tuner: \out{
Inherited methods} \itemize{ \item \out{}\href{../../mlr3tuning/html/Tuner.html#method-format}{\code{mlr3tuning::Tuner$format()}}\out{} +\item \out{}\href{../../mlr3tuning/html/Tuner.html#method-optimize}{\code{mlr3tuning::Tuner$optimize()}}\out{} \item \out{}\href{../../mlr3tuning/html/Tuner.html#method-print}{\code{mlr3tuning::Tuner$print()}}\out{} -\item \out{}\href{../../mlr3tuning/html/Tuner.html#method-tune}{\code{mlr3tuning::Tuner$tune()}}\out{} } \out{
} } \if{html}{\out{
}} \if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-new}{}}} \subsection{Method \code{new()}}{ Creates a new instance of this \link[R6:R6Class]{R6} class. \subsection{Usage}{ @@ -99,6 +101,7 @@ Creates a new instance of this \link[R6:R6Class]{R6} class. } \if{html}{\out{
}} \if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-clone}{}}} \subsection{Method \code{clone()}}{ The objects of this class are cloneable with this method. \subsection{Usage}{ diff --git a/tests/testthat/helper.R b/tests/testthat/helper.R index 3095027f8..f4aea205f 100644 --- a/tests/testthat/helper.R +++ b/tests/testthat/helper.R @@ -226,10 +226,12 @@ MAKE_GL = function() { } # mlr3::mlr_learners$add("regr.depparams", LearnerRegrDepParams) -expect_irace_parameters = function(parameters, names, types, domain, conditions, hierarchy){ - expect_list(parameters, len = 11, any.missing = FALSE) +expect_irace_parameters = function(parameters, names, types, domain, conditions, depends, + hierarchy){ + expect_list(parameters, len = 12, any.missing = FALSE) expect_equal(names(parameters), c("names", "types", "switches", "domain", "conditions","isFixed", - "transform","hierarchy","nbParameters","nbFixed","nbVariable")) + "transform","depends", "hierarchy","nbParameters","nbFixed", + "nbVariable")) expect_equal(parameters$names, names) expect_equal(parameters$types, set_names(types, names)) expect_equal(parameters$switches, named_vector(names, "")) @@ -242,6 +244,11 @@ expect_irace_parameters = function(parameters, names, types, domain, conditions, } expect_equal(parameters$isFixed, named_vector(names, FALSE)) expect_equal(parameters$transform, named_list(names, "")) + if (missing(depends)) { + expect_equal(parameters$depends, named_list(names, character(0))) + } else { + expect_equal(parameters$depends, depends) + } if (missing(hierarchy)) { expect_equal(parameters$hierarchy, named_vector(names, 1)) } else { diff --git a/tests/testthat/test_TunerIrace.R b/tests/testthat/test_TunerIrace.R index 1798882a9..e8eaf0f40 100644 --- a/tests/testthat/test_TunerIrace.R +++ b/tests/testthat/test_TunerIrace.R @@ -1,11 +1,10 @@ -context("TunerIrace") library(mlr3learners) set.seed(1) skip_if_not_installed("irace") test_that("TunerIrace", { - test_tuner("irace", term_evals = 38) + test_tuner("irace", term_evals = 42, real_evals = 39) }) test_that("TunerIrace with int params and trafo, clock terminator", { @@ -17,37 +16,35 @@ test_that("TunerIrace with int params and trafo, clock terminator", { x$minsplit = as.integer(round(x$minsplit)) return(x) } - inst = TuningInstance$new(tsk("iris"), lrn("classif.rpart"), rsmp("holdout"), msr("classif.ce"), - ps, term("clock_time", secs = 10)) + inst = TuningInstanceSingleCrit$new(tsk("iris"), lrn("classif.rpart"), rsmp("holdout"), + msr("classif.ce"), trm("run_time", secs = 24), ps) tt = tnr("irace") - tt$tune(inst) - d = inst$archive(unnest = "params") - expect_integer(d$minsplit) + tt$optimize(inst) + expect_double(inst$archive$best()$cp) }) test_that("TunerIrace with dependencies",{ ps = ParamSet$new(params = list( - ParamDbl$new("cost", lower = 0, upper = 1), - ParamFct$new("kernel", levels = c("linear","polynomial")) + ParamDbl$new("cp", lower = 0.001, upper = 0.1), + ParamInt$new("minsplit", lower = 1, upper = 10) )) - ps$add_dep("cost","kernel",CondEqual$new("polynomial")) - inst = TuningInstance$new(tsk("iris"), lrn("classif.svm",type="C-classification"), - rsmp("holdout"), msr("classif.ce"), ps, term("clock_time", secs = 8)) + ps$add_dep("minsplit","cp",CondEqual$new(0.005)) + inst = TuningInstanceSingleCrit$new(tsk("iris"), lrn("classif.rpart"), + rsmp("holdout"), msr("classif.ce"), + trm("evals", n_evals = 96), ps) tt = tnr("irace") - # should it possible to return kernel = linear...? - expect_silent(suppressWarnings(tt$tune(inst))) + expect_output(tt$optimize(inst)) }) test_that("minimize time",{ ps = ParamSet$new(params = list( - ParamDbl$new("cp", lower = 0.001, upper = 0.1), - ParamInt$new("minsplit", lower = 1, upper = 10) + ParamDbl$new("cp", lower = 0.001, upper = 0.1) )) - te = term("clock_time", secs = 11) - inst = TuningInstance$new(tsk("iris"), lrn("classif.rpart"), rsmp("holdout"), msr("classif.ce"), ps, te) - tt = tnr("irace", capping = 1, boundMax = 1, cappingType = "best", boundType= "instance") - tt$tune(inst) - expect_double(inst$archive(unnest = "params")$minsplit) + inst = TuningInstanceSingleCrit$new(tsk("iris"), lrn("classif.rpart"), rsmp("holdout"), + msr("classif.ce"), trm("run_time", secs = 20), ps) + tt = tnr("irace", capping = 1, boundMax = 1, cappingType = "best", boundType = "instance") + tt$optimize(inst) + expect_double(inst$archive$best()$cp) }) test_that("paradox_to_irace no dependencies",{ @@ -98,6 +95,7 @@ test_that("paradox_to_irace dependencies",{ expect_irace_parameters(parameters = paradox_to_irace(ps), names = c("a","b"), types = c("c","i"), domain = list(a = c("TRUE", "FALSE"), b = c(1,9)), conditions = list(a = TRUE, b = expression(a == TRUE)), + depends = list(a = character(0), b = "a"), hierarchy = c(1,2)) ps = ParamSet$new(params = list( @@ -110,7 +108,9 @@ test_that("paradox_to_irace dependencies",{ expect_irace_parameters(parameters = paradox_to_irace(ps), names = c("a","c","b"), types = c("c","c","i"), domain = list(a = c("TRUE", "FALSE"), c = c("lvl1","lvl2"), b = c(1,9)), - conditions = list(a = TRUE, b = expression(a == TRUE), c = expression(b %in% c(2,5,7))), + conditions = list(a = TRUE, b = expression(a == TRUE), + c = expression(b %in% c(2,5,7))), + depends = list(a = character(0),c = "b", b = "a"), hierarchy = c(1,3,2)) @@ -125,60 +125,65 @@ test_that("paradox_to_irace dependencies",{ ps$add_dep("d", "c", CondAnyOf$new(c("lvl1","lvl2"))) expect_irace_parameters(parameters = paradox_to_irace(ps), names = c("a","b","c","d"), types = c("c","i","c","r"), - domain = list(a = c("TRUE", "FALSE"), b = c(1,9), c = c("lvl1","lvl2"), d = c(0,1)), + domain = list(a = c("TRUE", "FALSE"), b = c(1,9), c = c("lvl1","lvl2"), + d = c(0,1)), conditions = list(c = TRUE, a = expression(c == "lvl1"), - d = expression(c %in% c('lvl1', 'lvl2')),b = expression(a == TRUE)), + d = expression(c %in% c('lvl1', 'lvl2')), + b = expression(a == TRUE)), + depends = list(a = "c", b = "a", c = character(0), d = "c"), hierarchy = c(2,3,1,2)) }) test_that("TunerIrace works with logical params", { ps = ParamSet$new(params = list( - ParamLgl$new("intercept") + ParamLgl$new("keep_model") )) - inst = TuningInstance$new(tsk("mtcars"), lrn("regr.glmnet"), rsmp("holdout"), msr("regr.mse"), - ps, term("clock_time", secs = 10)) + inst = TuningInstanceSingleCrit$new(tsk("mtcars"), lrn("regr.rpart"), rsmp("holdout"), + msr("regr.mse"), + trm("evals", n_evals = 42), ps) tt = tnr("irace") - expect_warning(tt$tune(inst)) - d = inst$archive(unnest = "params") - expect_logical(d$intercept) + tt$optimize(inst) + expect_logical(inst$archive$best()$keep_model) }) test_that("TunerIrace works with tune.threshold", { ps = ParamSet$new(params = list( ParamInt$new("minsplit", lower = 1, upper = 3) )) - te = term("evals", n_evals = 50) tt = tnr("irace", nbIterations = 1L, minNbSurvival = 1) - inst = TuningInstance$new(tsk("iris"), lrn("classif.rpart"), rsmp("holdout", ratio = 0.1), msr("classif.ce"), ps, te) - expect_silent(tt$tune(inst)) + inst = TuningInstanceSingleCrit$new(tsk("iris"), lrn("classif.rpart"), + rsmp("holdout", ratio = 0.1), msr("classif.ce"), + trm("evals", n_evals = 50), ps) + expect_output(tt$optimize(inst)) }) test_that("TunerIrace uses digits", { - te = term("evals", n_evals = 30) tt = tnr("irace", nbIterations = 1L, minNbSurvival = 1) ps = ParamSet$new(params = list( - ParamDbl$new("cost", lower = pi * 1e-20,upper = 5.242e12) + ParamDbl$new("cp", lower = pi * 1e-20, upper = 5.242e12/1e13) )) - inst = TuningInstance$new(tsk("iris"), lrn("classif.svm", type="C-classification"), rsmp("holdout"), msr("classif.ce"), ps, te) - expect_silent(tt$tune(inst)) + inst = TuningInstanceSingleCrit$new(tsk("iris"), lrn("classif.rpart"), rsmp("holdout"), + msr("classif.ce"), trm("evals", n_evals = 30), ps) + expect_output(tt$optimize(inst)) ps = ParamSet$new(params = list( ParamDbl$new("cp", lower = 1e-5, upper = 1e-4) )) - te = term("evals", n_evals = 40) tt = tnr("irace", digits = 3L) - inst = TuningInstance$new(tsk("iris"), lrn("classif.rpart"), rsmp("holdout"), msr("classif.ce"), ps, te) - expect_error(tt$tune(inst)) + inst = TuningInstanceSingleCrit$new(tsk("iris"), lrn("classif.rpart"), rsmp("holdout"), + msr("classif.ce"), trm("evals", n_evals = 40), ps) + expect_error(tt$optimize(inst)) }) test_that("Error in hyperparameter tuning with scientific notation for lower/upper boundaries", { ps = ParamSet$new(params = list( ParamDbl$new("cp", lower = 1e-3, upper = 1e-1) )) - inst = TuningInstance$new(tsk("iris"), lrn("classif.rpart"), rsmp("holdout"), msr("classif.ce"), ps, term("evals", n_evals = 30)) + inst = TuningInstanceSingleCrit$new(tsk("iris"), lrn("classif.rpart"), rsmp("holdout"), + msr("classif.ce"), trm("evals", n_evals = 30), ps) tt = tnr("irace", nbIterations = 1L) - expect_silent(tt$tune(inst)) + expect_output(tt$optimize(inst)) }) # we had a bug here, see (mlr) issue #627 @@ -186,23 +191,8 @@ test_that("irace works with unnamed discrete values", { ps = ParamSet$new(params = list( ParamInt$new("minsplit", lower = 2L, upper = 7L) )) - inst = TuningInstance$new(tsk("iris"), lrn("classif.rpart"), rsmp("holdout"), msr("classif.ce"), - ps, term("evals", n_evals = 50)) + inst = TuningInstanceSingleCrit$new(tsk("iris"), lrn("classif.rpart"), rsmp("holdout"), msr("classif.ce"), + trm("evals", n_evals = 50), ps) tt = tnr("irace") - expect_silent(suppressWarnings(tt$tune(inst))) -}) - -# shouldn't paradox check this? -test_that("irace handles parameters with unsatisfiable requirement gracefully", { - skip_on_os("windows") - tt = tnr("irace", nbIterations = 1L, minNbSurvival = 1L) - - ps = ParamSet$new(params = list( - ParamDbl$new("cost", lower = 0, upper = 1), - ParamFct$new("kernel", levels = c("linear","polynomial")) - )) - ps$add_dep("kernel","cost",CondEqual$new(-1)) # cost never feasible - inst = TuningInstance$new(tsk("iris"), lrn("classif.svm", type= "C-classification"), rsmp("holdout"), msr("classif.ce"), - ps, term("evals", n_evals = 50)) - expect_silent(suppressWarnings(tt$tune(inst))) + expect_output(tt$optimize(inst)) }) From fbc4924e9b68fb323f6907d850e2b5e69858594c Mon Sep 17 00:00:00 2001 From: Raphael Sonabend Date: Tue, 8 Dec 2020 22:25:27 +0000 Subject: [PATCH 12/40] styler --- R/TunerIrace.R | 51 ++++++----- R/irace_helpers.R | 34 +++---- tests/testthat/test_TunerIrace.R | 152 +++++++++++++++++-------------- 3 files changed, 131 insertions(+), 106 deletions(-) diff --git a/R/TunerIrace.R b/R/TunerIrace.R index 11f59a6fb..be57f2a85 100644 --- a/R/TunerIrace.R +++ b/R/TunerIrace.R @@ -50,7 +50,8 @@ #' @export #' @examples #' # see ?Tuner -TunerIrace = R6Class("TunerIrace", inherit = Tuner, +TunerIrace = R6Class("TunerIrace", + inherit = Tuner, public = list( #' @description #' Creates a new instance of this [R6][R6::R6Class] class. @@ -59,35 +60,35 @@ TunerIrace = R6Class("TunerIrace", inherit = Tuner, ParamLgl$new("show.irace.output", default = FALSE), ParamInt$new("debugLevel", default = 0, lower = 0), ParamInt$new("seed"), - ParamDbl$new("postselection",default=0,lower=0,upper=1), - ParamInt$new("elitist",default=1,lower=0,upper=1), - ParamInt$new("elitistLimit",default=2,lower=0), - ParamInt$new("nbIterations",default=0,lower=0), - ParamInt$new("nbExperimentsPerIteration",default=0,lower=0), - ParamInt$new("minNbSurvival",default=0,lower=0), - ParamInt$new("nbConfigurations",default=0,lower=0), - ParamInt$new("mu",default=5,lower=1), - ParamInt$new("softRestart",default=1,lower=0,upper=1), + ParamDbl$new("postselection", default = 0, lower = 0, upper = 1), + ParamInt$new("elitist", default = 1, lower = 0, upper = 1), + ParamInt$new("elitistLimit", default = 2, lower = 0), + ParamInt$new("nbIterations", default = 0, lower = 0), + ParamInt$new("nbExperimentsPerIteration", default = 0, lower = 0), + ParamInt$new("minNbSurvival", default = 0, lower = 0), + ParamInt$new("nbConfigurations", default = 0, lower = 0), + ParamInt$new("mu", default = 5, lower = 1), + ParamInt$new("softRestart", default = 1, lower = 0, upper = 1), ParamDbl$new("softRestartThreshold"), ParamInt$new("digits", default = 4, lower = 1, upper = 15), - ParamFct$new("testType", default = "F-test", levels = c("F-test","t-test","t-test-bonferroni","t-test-holm")), + ParamFct$new("testType", default = "F-test", levels = c("F-test", "t-test", "t-test-bonferroni", "t-test-holm")), ParamInt$new("firstTest", default = 5, lower = 0), - ParamInt$new("eachTest",default = 1, lower = 1), - ParamDbl$new("confidence",default=0.95,lower=0,upper=1), - ParamInt$new("capping",default = 0,lower=0,upper=1), - ParamFct$new("cappingType",default="median",levels=c("median","mean","best","worst")), - ParamFct$new("boundType",default="candidate",levels=c("candidate","instance")), - ParamDbl$new("boundMax",default=0), - ParamInt$new("boundDigits",default=0), - ParamDbl$new("boundPar",default=1), - ParamDbl$new("boundAsTimeout",default=1) + ParamInt$new("eachTest", default = 1, lower = 1), + ParamDbl$new("confidence", default = 0.95, lower = 0, upper = 1), + ParamInt$new("capping", default = 0, lower = 0, upper = 1), + ParamFct$new("cappingType", default = "median", levels = c("median", "mean", "best", "worst")), + ParamFct$new("boundType", default = "candidate", levels = c("candidate", "instance")), + ParamDbl$new("boundMax", default = 0), + ParamInt$new("boundDigits", default = 0), + ParamDbl$new("boundPar", default = 1), + ParamDbl$new("boundAsTimeout", default = 1) )) ps$values$show.irace.output = FALSE super$initialize( param_set = ps, - param_classes = c("ParamDbl","ParamInt","ParamFct","ParamLgl"), + param_classes = c("ParamDbl", "ParamInt", "ParamFct", "ParamLgl"), properties = c("dependencies", "single-crit"), packages = "irace" ) @@ -97,9 +98,11 @@ TunerIrace = R6Class("TunerIrace", inherit = Tuner, private = list( .optimize = function(inst) { g = if (self$param_set$values$show.irace.output) identity else capture.output - g(irace::irace(scenario = c(make_scenario(inst), - self$param_set$values[names(self$param_set$values) %nin% "show.irace.output"]), - parameters = paradox_to_irace(inst$search_space))) + g(irace::irace( + scenario = c( + make_scenario(inst), + self$param_set$values[names(self$param_set$values) %nin% "show.irace.output"]), + parameters = paradox_to_irace(inst$search_space))) } ) ) diff --git a/R/irace_helpers.R b/R/irace_helpers.R index ea81b296f..38cf8e81c 100644 --- a/R/irace_helpers.R +++ b/R/irace_helpers.R @@ -1,11 +1,13 @@ -paradox_to_irace = function(ps){ +paradox_to_irace = function(ps) { + assertClass(ps, "ParamSet") # what about ParamUty = vector numeric/real - class_lookup = data.table(paradox = c("ParamLgl","ParamInt","ParamDbl","ParamFct"), - irace = c("c","i","r","c"), stringsAsFactors = FALSE) + class_lookup = data.table( + paradox = c("ParamLgl", "ParamInt", "ParamDbl", "ParamFct"), + irace = c("c", "i", "r", "c"), stringsAsFactors = FALSE) type = unlist(subset(merge(data.table(paradox = ps$class), class_lookup, sort = FALSE), - select = "irace")) + select = "irace")) range = get_irace_range(ps) if (ps$has_deps) { condition = get_irace_condition(ps) @@ -17,28 +19,28 @@ paradox_to_irace = function(ps){ return(irace::readParameters(text = par.tab)) } -get_irace_range = function(ps){ +get_irace_range = function(ps) { rng = data.table(lower = ps$lower, upper = ps$upper, lvl = ps$levels) - apply(rng, 1, function(x){ + apply(rng, 1, function(x) { if (is.na(x[[1]])) { - return(paste0("(",paste0(x[[3]], collapse = ","),")")) + return(paste0("(", paste0(x[[3]], collapse = ","), ")")) } else { - return(paste0("(",x[[1]],",",x[[2]],")")) + return(paste0("(", x[[1]], ",", x[[2]], ")")) } }) } -get_irace_condition = function(ps){ - cond = rbindlist(apply(ps$deps, 1, function(x){ +get_irace_condition = function(ps) { + cond = rbindlist(apply(ps$deps, 1, function(x) { on = x[[2]] cond = x[[3]]$rhs if (is.character(cond)) { - cond = paste0("'", cond ,"'") + cond = paste0("'", cond, "'") } if (x[[3]]$type == "equal") { - condition = paste("|",x[[2]], "==", cond) + condition = paste("|", x[[2]], "==", cond) } else { - condition = paste("|",x[[2]],"%in%", paste0("c(",paste0(cond, collapse = ","),")")) + condition = paste("|", x[[2]], "%in%", paste0("c(", paste0(cond, collapse = ","), ")")) } data.table(id = x[[1]], cond = condition) })) @@ -49,7 +51,7 @@ get_irace_condition = function(ps){ return(tab) } -make_scenario = function(inst){ +make_scenario = function(inst) { list( targetRunner = targetRunner, logFile = tempfile(), @@ -60,10 +62,10 @@ make_scenario = function(inst){ ) } -targetRunner = function(experiment, scenario){ +targetRunner = function(experiment, scenario) { t0 = Sys.time() # fix logicals - config = as.data.table(lapply(experiment$configuration, function(x){ + config = as.data.table(lapply(experiment$configuration, function(x) { if (x %in% c("TRUE", "FALSE")) { return(as.logical(x)) } else { diff --git a/tests/testthat/test_TunerIrace.R b/tests/testthat/test_TunerIrace.R index e8eaf0f40..61505cca4 100644 --- a/tests/testthat/test_TunerIrace.R +++ b/tests/testthat/test_TunerIrace.R @@ -16,43 +16,47 @@ test_that("TunerIrace with int params and trafo, clock terminator", { x$minsplit = as.integer(round(x$minsplit)) return(x) } - inst = TuningInstanceSingleCrit$new(tsk("iris"), lrn("classif.rpart"), rsmp("holdout"), - msr("classif.ce"), trm("run_time", secs = 24), ps) + inst = TuningInstanceSingleCrit$new( + tsk("iris"), lrn("classif.rpart"), rsmp("holdout"), + msr("classif.ce"), trm("run_time", secs = 24), ps) tt = tnr("irace") tt$optimize(inst) expect_double(inst$archive$best()$cp) }) -test_that("TunerIrace with dependencies",{ +test_that("TunerIrace with dependencies", { ps = ParamSet$new(params = list( ParamDbl$new("cp", lower = 0.001, upper = 0.1), ParamInt$new("minsplit", lower = 1, upper = 10) )) - ps$add_dep("minsplit","cp",CondEqual$new(0.005)) - inst = TuningInstanceSingleCrit$new(tsk("iris"), lrn("classif.rpart"), - rsmp("holdout"), msr("classif.ce"), - trm("evals", n_evals = 96), ps) + ps$add_dep("minsplit", "cp", CondEqual$new(0.005)) + inst = TuningInstanceSingleCrit$new( + tsk("iris"), lrn("classif.rpart"), + rsmp("holdout"), msr("classif.ce"), + trm("evals", n_evals = 96), ps) tt = tnr("irace") expect_output(tt$optimize(inst)) }) -test_that("minimize time",{ +test_that("minimize time", { ps = ParamSet$new(params = list( ParamDbl$new("cp", lower = 0.001, upper = 0.1) )) - inst = TuningInstanceSingleCrit$new(tsk("iris"), lrn("classif.rpart"), rsmp("holdout"), - msr("classif.ce"), trm("run_time", secs = 20), ps) + inst = TuningInstanceSingleCrit$new( + tsk("iris"), lrn("classif.rpart"), rsmp("holdout"), + msr("classif.ce"), trm("run_time", secs = 20), ps) tt = tnr("irace", capping = 1, boundMax = 1, cappingType = "best", boundType = "instance") tt$optimize(inst) expect_double(inst$archive$best()$cp) }) -test_that("paradox_to_irace no dependencies",{ +test_that("paradox_to_irace no dependencies", { ps = ParamSet$new(params = list( ParamLgl$new("lgl") )) - expect_irace_parameters(parameters = paradox_to_irace(ps), names = "lgl", types = "c", - domain = list(lgl = c("TRUE", "FALSE")), conditions = list(lgl = TRUE)) + expect_irace_parameters( + parameters = paradox_to_irace(ps), names = "lgl", types = "c", + domain = list(lgl = c("TRUE", "FALSE")), conditions = list(lgl = TRUE)) ps = ParamSet$new(params = list( ParamUty$new("uty") @@ -62,85 +66,96 @@ test_that("paradox_to_irace no dependencies",{ ps = ParamSet$new(params = list( ParamDbl$new("dbl", lower = 0.1, upper = 0.3), ParamInt$new("int", lower = 1, upper = 9), - ParamFct$new("fct", levels = c("a","b","c")), + ParamFct$new("fct", levels = c("a", "b", "c")), ParamLgl$new("lgl") )) - expect_irace_parameters(parameters = paradox_to_irace(ps), - names = c("dbl","int","fct","lgl"), - types = c("r","i","c","c"), - domain = list(dbl = c(0.1,0.3), int = c(1,9), fct = c("a","b","c"), - lgl = c("TRUE", "FALSE"))) + expect_irace_parameters( + parameters = paradox_to_irace(ps), + names = c("dbl", "int", "fct", "lgl"), + types = c("r", "i", "c", "c"), + domain = list( + dbl = c(0.1, 0.3), int = c(1, 9), fct = c("a", "b", "c"), + lgl = c("TRUE", "FALSE"))) # double checking previous bug in merge sort ps = ParamSet$new(params = list( - ParamFct$new("fct", levels = c("a","b","c")), + ParamFct$new("fct", levels = c("a", "b", "c")), ParamInt$new("int1", lower = 1, upper = 9), ParamDbl$new("dbl", lower = 0.1, upper = 0.3), ParamInt$new("int2", lower = 10, upper = 90), ParamLgl$new("lgl") )) - expect_irace_parameters(parameters = paradox_to_irace(ps), - names = c("fct","int1","dbl","int2","lgl"), - types = c("c","i","r","i","c"), - domain = list(fct = c("a","b","c"), int1 = c(1,9), dbl = c(0.1,0.3), - int2 = c(10,90), lgl = c("TRUE", "FALSE"))) + expect_irace_parameters( + parameters = paradox_to_irace(ps), + names = c("fct", "int1", "dbl", "int2", "lgl"), + types = c("c", "i", "r", "i", "c"), + domain = list( + fct = c("a", "b", "c"), int1 = c(1, 9), dbl = c(0.1, 0.3), + int2 = c(10, 90), lgl = c("TRUE", "FALSE"))) }) -test_that("paradox_to_irace dependencies",{ +test_that("paradox_to_irace dependencies", { ps = ParamSet$new(params = list( ParamLgl$new("a"), ParamInt$new("b", lower = 1, upper = 9)) ) ps$add_dep("b", "a", CondEqual$new(TRUE)) - expect_irace_parameters(parameters = paradox_to_irace(ps), names = c("a","b"), types = c("c","i"), - domain = list(a = c("TRUE", "FALSE"), b = c(1,9)), - conditions = list(a = TRUE, b = expression(a == TRUE)), - depends = list(a = character(0), b = "a"), - hierarchy = c(1,2)) + expect_irace_parameters( + parameters = paradox_to_irace(ps), names = c("a", "b"), types = c("c", "i"), + domain = list(a = c("TRUE", "FALSE"), b = c(1, 9)), + conditions = list(a = TRUE, b = expression(a == TRUE)), + depends = list(a = character(0), b = "a"), + hierarchy = c(1, 2)) ps = ParamSet$new(params = list( ParamLgl$new("a"), - ParamFct$new("c", levels = c("lvl1","lvl2")), + ParamFct$new("c", levels = c("lvl1", "lvl2")), ParamInt$new("b", lower = 1, upper = 9) )) ps$add_dep("b", "a", CondEqual$new(TRUE)) - ps$add_dep("c", "b", CondAnyOf$new(c(2,5,7))) - expect_irace_parameters(parameters = paradox_to_irace(ps), names = c("a","c","b"), - types = c("c","c","i"), - domain = list(a = c("TRUE", "FALSE"), c = c("lvl1","lvl2"), b = c(1,9)), - conditions = list(a = TRUE, b = expression(a == TRUE), - c = expression(b %in% c(2,5,7))), - depends = list(a = character(0),c = "b", b = "a"), - hierarchy = c(1,3,2)) + ps$add_dep("c", "b", CondAnyOf$new(c(2, 5, 7))) + expect_irace_parameters( + parameters = paradox_to_irace(ps), names = c("a", "c", "b"), + types = c("c", "c", "i"), + domain = list(a = c("TRUE", "FALSE"), c = c("lvl1", "lvl2"), b = c(1, 9)), + conditions = list( + a = TRUE, b = expression(a == TRUE), + c = expression(b %in% c(2, 5, 7))), + depends = list(a = character(0), c = "b", b = "a"), + hierarchy = c(1, 3, 2)) ps = ParamSet$new(params = list( ParamLgl$new("a"), ParamInt$new("b", lower = 1, upper = 9), - ParamFct$new("c", levels = c("lvl1","lvl2")), + ParamFct$new("c", levels = c("lvl1", "lvl2")), ParamDbl$new("d", lower = 0, upper = 1) )) ps$add_dep("b", "a", CondEqual$new(TRUE)) ps$add_dep("a", "c", CondEqual$new("lvl1")) - ps$add_dep("d", "c", CondAnyOf$new(c("lvl1","lvl2"))) - expect_irace_parameters(parameters = paradox_to_irace(ps), names = c("a","b","c","d"), - types = c("c","i","c","r"), - domain = list(a = c("TRUE", "FALSE"), b = c(1,9), c = c("lvl1","lvl2"), - d = c(0,1)), - conditions = list(c = TRUE, a = expression(c == "lvl1"), - d = expression(c %in% c('lvl1', 'lvl2')), - b = expression(a == TRUE)), - depends = list(a = "c", b = "a", c = character(0), d = "c"), - hierarchy = c(2,3,1,2)) + ps$add_dep("d", "c", CondAnyOf$new(c("lvl1", "lvl2"))) + expect_irace_parameters( + parameters = paradox_to_irace(ps), names = c("a", "b", "c", "d"), + types = c("c", "i", "c", "r"), + domain = list( + a = c("TRUE", "FALSE"), b = c(1, 9), c = c("lvl1", "lvl2"), + d = c(0, 1)), + conditions = list( + c = TRUE, a = expression(c == "lvl1"), + d = expression(c %in% c("lvl1", "lvl2")), + b = expression(a == TRUE)), + depends = list(a = "c", b = "a", c = character(0), d = "c"), + hierarchy = c(2, 3, 1, 2)) }) test_that("TunerIrace works with logical params", { ps = ParamSet$new(params = list( ParamLgl$new("keep_model") )) - inst = TuningInstanceSingleCrit$new(tsk("mtcars"), lrn("regr.rpart"), rsmp("holdout"), - msr("regr.mse"), - trm("evals", n_evals = 42), ps) + inst = TuningInstanceSingleCrit$new( + tsk("mtcars"), lrn("regr.rpart"), rsmp("holdout"), + msr("regr.mse"), + trm("evals", n_evals = 42), ps) tt = tnr("irace") tt$optimize(inst) expect_logical(inst$archive$best()$keep_model) @@ -151,19 +166,21 @@ test_that("TunerIrace works with tune.threshold", { ParamInt$new("minsplit", lower = 1, upper = 3) )) tt = tnr("irace", nbIterations = 1L, minNbSurvival = 1) - inst = TuningInstanceSingleCrit$new(tsk("iris"), lrn("classif.rpart"), - rsmp("holdout", ratio = 0.1), msr("classif.ce"), - trm("evals", n_evals = 50), ps) + inst = TuningInstanceSingleCrit$new( + tsk("iris"), lrn("classif.rpart"), + rsmp("holdout", ratio = 0.1), msr("classif.ce"), + trm("evals", n_evals = 50), ps) expect_output(tt$optimize(inst)) }) test_that("TunerIrace uses digits", { tt = tnr("irace", nbIterations = 1L, minNbSurvival = 1) ps = ParamSet$new(params = list( - ParamDbl$new("cp", lower = pi * 1e-20, upper = 5.242e12/1e13) + ParamDbl$new("cp", lower = pi * 1e-20, upper = 5.242e12 / 1e13) )) - inst = TuningInstanceSingleCrit$new(tsk("iris"), lrn("classif.rpart"), rsmp("holdout"), - msr("classif.ce"), trm("evals", n_evals = 30), ps) + inst = TuningInstanceSingleCrit$new( + tsk("iris"), lrn("classif.rpart"), rsmp("holdout"), + msr("classif.ce"), trm("evals", n_evals = 30), ps) expect_output(tt$optimize(inst)) @@ -171,8 +188,9 @@ test_that("TunerIrace uses digits", { ParamDbl$new("cp", lower = 1e-5, upper = 1e-4) )) tt = tnr("irace", digits = 3L) - inst = TuningInstanceSingleCrit$new(tsk("iris"), lrn("classif.rpart"), rsmp("holdout"), - msr("classif.ce"), trm("evals", n_evals = 40), ps) + inst = TuningInstanceSingleCrit$new( + tsk("iris"), lrn("classif.rpart"), rsmp("holdout"), + msr("classif.ce"), trm("evals", n_evals = 40), ps) expect_error(tt$optimize(inst)) }) @@ -180,8 +198,9 @@ test_that("Error in hyperparameter tuning with scientific notation for lower/upp ps = ParamSet$new(params = list( ParamDbl$new("cp", lower = 1e-3, upper = 1e-1) )) - inst = TuningInstanceSingleCrit$new(tsk("iris"), lrn("classif.rpart"), rsmp("holdout"), - msr("classif.ce"), trm("evals", n_evals = 30), ps) + inst = TuningInstanceSingleCrit$new( + tsk("iris"), lrn("classif.rpart"), rsmp("holdout"), + msr("classif.ce"), trm("evals", n_evals = 30), ps) tt = tnr("irace", nbIterations = 1L) expect_output(tt$optimize(inst)) }) @@ -191,8 +210,9 @@ test_that("irace works with unnamed discrete values", { ps = ParamSet$new(params = list( ParamInt$new("minsplit", lower = 2L, upper = 7L) )) - inst = TuningInstanceSingleCrit$new(tsk("iris"), lrn("classif.rpart"), rsmp("holdout"), msr("classif.ce"), - trm("evals", n_evals = 50), ps) + inst = TuningInstanceSingleCrit$new( + tsk("iris"), lrn("classif.rpart"), rsmp("holdout"), msr("classif.ce"), + trm("evals", n_evals = 50), ps) tt = tnr("irace") expect_output(tt$optimize(inst)) }) From 3787178cb61bd44612d6d96913e3ae627ebbb11b Mon Sep 17 00:00:00 2001 From: Raphael Sonabend Date: Tue, 8 Dec 2020 22:28:16 +0000 Subject: [PATCH 13/40] lint --- R/TunerIrace.R | 5 +++-- R/irace_helpers.R | 12 +++++++----- tests/testthat/helper.R | 6 +++--- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/R/TunerIrace.R b/R/TunerIrace.R index be57f2a85..96b55bdd5 100644 --- a/R/TunerIrace.R +++ b/R/TunerIrace.R @@ -71,7 +71,8 @@ TunerIrace = R6Class("TunerIrace", ParamInt$new("softRestart", default = 1, lower = 0, upper = 1), ParamDbl$new("softRestartThreshold"), ParamInt$new("digits", default = 4, lower = 1, upper = 15), - ParamFct$new("testType", default = "F-test", levels = c("F-test", "t-test", "t-test-bonferroni", "t-test-holm")), + ParamFct$new("testType", default = "F-test", + levels = c("F-test", "t-test", "t-test-bonferroni", "t-test-holm")), ParamInt$new("firstTest", default = 5, lower = 0), ParamInt$new("eachTest", default = 1, lower = 1), ParamDbl$new("confidence", default = 0.95, lower = 0, upper = 1), @@ -84,7 +85,7 @@ TunerIrace = R6Class("TunerIrace", ParamDbl$new("boundAsTimeout", default = 1) )) - ps$values$show.irace.output = FALSE + ps$values$show.irace.output = FALSE # nolint super$initialize( param_set = ps, diff --git a/R/irace_helpers.R b/R/irace_helpers.R index 38cf8e81c..98b1e97a4 100644 --- a/R/irace_helpers.R +++ b/R/irace_helpers.R @@ -15,9 +15,9 @@ paradox_to_irace = function(ps) { condition = NULL } - par.tab = paste(ps$ids(), '""', type, range, condition$cond, collapse = "\n") + par_tab = paste(ps$ids(), '""', type, range, condition$cond, collapse = "\n") - return(irace::readParameters(text = par.tab)) + return(irace::readParameters(text = par_tab)) } get_irace_range = function(ps) { rng = data.table(lower = ps$lower, upper = ps$upper, lvl = ps$levels) @@ -57,12 +57,14 @@ make_scenario = function(inst) { logFile = tempfile(), instances = list(inst), debugLevel = 0, - maxExperiments = if (class(inst$terminator)[1] == "TerminatorEvals") inst$terminator$param_set$values$n_evals else 0, - maxTime = if (class(inst$terminator)[1] == "TerminatorRunTime") inst$terminator$param_set$values$secs else 0 + maxExperiments = if (class(inst$terminator)[1] == "TerminatorEvals") + inst$terminator$param_set$values$n_evals else 0, + maxTime = if (class(inst$terminator)[1] == "TerminatorRunTime") + inst$terminator$param_set$values$secs else 0 ) } -targetRunner = function(experiment, scenario) { +targetRunner = function(experiment, scenario) { # nolint t0 = Sys.time() # fix logicals config = as.data.table(lapply(experiment$configuration, function(x) { diff --git a/tests/testthat/helper.R b/tests/testthat/helper.R index f4aea205f..25511168b 100644 --- a/tests/testthat/helper.R +++ b/tests/testthat/helper.R @@ -227,10 +227,10 @@ MAKE_GL = function() { # mlr3::mlr_learners$add("regr.depparams", LearnerRegrDepParams) expect_irace_parameters = function(parameters, names, types, domain, conditions, depends, - hierarchy){ + hierarchy) { expect_list(parameters, len = 12, any.missing = FALSE) - expect_equal(names(parameters), c("names", "types", "switches", "domain", "conditions","isFixed", - "transform","depends", "hierarchy","nbParameters","nbFixed", + expect_equal(names(parameters), c("names", "types", "switches", "domain", "conditions", "isFixed", + "transform", "depends", "hierarchy", "nbParameters", "nbFixed", "nbVariable")) expect_equal(parameters$names, names) expect_equal(parameters$types, set_names(types, names)) From 04dfc024586f9dab728a5bcdd5bab1ccff9675e7 Mon Sep 17 00:00:00 2001 From: Raphael Sonabend Date: Tue, 8 Dec 2020 22:37:28 +0000 Subject: [PATCH 14/40] clear warnings --- R/TunerIrace.R | 2 +- tests/testthat/test_TunerIrace.R | 24 +++++++++++++++--------- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/R/TunerIrace.R b/R/TunerIrace.R index 96b55bdd5..5dc6677d1 100644 --- a/R/TunerIrace.R +++ b/R/TunerIrace.R @@ -98,7 +98,7 @@ TunerIrace = R6Class("TunerIrace", private = list( .optimize = function(inst) { - g = if (self$param_set$values$show.irace.output) identity else capture.output + g = if (self$param_set$values$show.irace.output) identity else utils::capture.output g(irace::irace( scenario = c( make_scenario(inst), diff --git a/tests/testthat/test_TunerIrace.R b/tests/testthat/test_TunerIrace.R index 61505cca4..18aff7488 100644 --- a/tests/testthat/test_TunerIrace.R +++ b/tests/testthat/test_TunerIrace.R @@ -20,7 +20,8 @@ test_that("TunerIrace with int params and trafo, clock terminator", { tsk("iris"), lrn("classif.rpart"), rsmp("holdout"), msr("classif.ce"), trm("run_time", secs = 24), ps) tt = tnr("irace") - tt$optimize(inst) + # suppressWarnings used throughout - looks like coming upstream + suppressWarnings(tt$optimize(inst)) expect_double(inst$archive$best()$cp) }) @@ -35,7 +36,8 @@ test_that("TunerIrace with dependencies", { rsmp("holdout"), msr("classif.ce"), trm("evals", n_evals = 96), ps) tt = tnr("irace") - expect_output(tt$optimize(inst)) + suppressWarnings(tt$optimize(inst)) + expect_double(inst$archive$best()$cp) }) test_that("minimize time", { @@ -46,7 +48,7 @@ test_that("minimize time", { tsk("iris"), lrn("classif.rpart"), rsmp("holdout"), msr("classif.ce"), trm("run_time", secs = 20), ps) tt = tnr("irace", capping = 1, boundMax = 1, cappingType = "best", boundType = "instance") - tt$optimize(inst) + suppressWarnings(tt$optimize(inst)) expect_double(inst$archive$best()$cp) }) @@ -157,7 +159,7 @@ test_that("TunerIrace works with logical params", { msr("regr.mse"), trm("evals", n_evals = 42), ps) tt = tnr("irace") - tt$optimize(inst) + suppressWarnings(tt$optimize(inst)) expect_logical(inst$archive$best()$keep_model) }) @@ -170,7 +172,8 @@ test_that("TunerIrace works with tune.threshold", { tsk("iris"), lrn("classif.rpart"), rsmp("holdout", ratio = 0.1), msr("classif.ce"), trm("evals", n_evals = 50), ps) - expect_output(tt$optimize(inst)) + suppressWarnings(tt$optimize(inst)) + expect_double(inst$archive$best()$minsplit) }) test_that("TunerIrace uses digits", { @@ -181,7 +184,8 @@ test_that("TunerIrace uses digits", { inst = TuningInstanceSingleCrit$new( tsk("iris"), lrn("classif.rpart"), rsmp("holdout"), msr("classif.ce"), trm("evals", n_evals = 30), ps) - expect_output(tt$optimize(inst)) + suppressWarnings(tt$optimize(inst)) + expect_double(inst$archive$best()$cp) ps = ParamSet$new(params = list( @@ -191,7 +195,7 @@ test_that("TunerIrace uses digits", { inst = TuningInstanceSingleCrit$new( tsk("iris"), lrn("classif.rpart"), rsmp("holdout"), msr("classif.ce"), trm("evals", n_evals = 40), ps) - expect_error(tt$optimize(inst)) + expect_error(suppressWarnings(tt$optimize(inst))) }) test_that("Error in hyperparameter tuning with scientific notation for lower/upper boundaries", { @@ -202,7 +206,8 @@ test_that("Error in hyperparameter tuning with scientific notation for lower/upp tsk("iris"), lrn("classif.rpart"), rsmp("holdout"), msr("classif.ce"), trm("evals", n_evals = 30), ps) tt = tnr("irace", nbIterations = 1L) - expect_output(tt$optimize(inst)) + suppressWarnings(tt$optimize(inst)) + expect_double(inst$archive$best()$cp) }) # we had a bug here, see (mlr) issue #627 @@ -214,5 +219,6 @@ test_that("irace works with unnamed discrete values", { tsk("iris"), lrn("classif.rpart"), rsmp("holdout"), msr("classif.ce"), trm("evals", n_evals = 50), ps) tt = tnr("irace") - expect_output(tt$optimize(inst)) + suppressWarnings(tt$optimize(inst)) + expect_double(inst$archive$best()$minsplit) }) From f84cb78ee9688cb108609c82b0132ee14dd6f901 Mon Sep 17 00:00:00 2001 From: Raphael Sonabend Date: Tue, 12 Jan 2021 13:00:06 +0000 Subject: [PATCH 15/40] Update irace_helpers.R --- R/irace_helpers.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/irace_helpers.R b/R/irace_helpers.R index 98b1e97a4..8f8f920d3 100644 --- a/R/irace_helpers.R +++ b/R/irace_helpers.R @@ -74,7 +74,7 @@ targetRunner = function(experiment, scenario) { # nolint return(x) } })) - cost = scenario$instances[[1]]$eval_batch(config)[[1]] + cost = scenario$instances[[1]]$objective_function(config)#[[1]] t1 = Sys.time() return(list(cost = cost, time = as.numeric(t1 - t0))) From 0c752fafa38b4dd7d12db1e76a012cf1a2217457 Mon Sep 17 00:00:00 2001 From: Marc Becker Date: Thu, 14 Jan 2021 11:46:15 +0100 Subject: [PATCH 16/40] Make Objective$resampling an active binding --- R/ObjectiveTuning.R | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/R/ObjectiveTuning.R b/R/ObjectiveTuning.R index e67c343c8..be100c88d 100644 --- a/R/ObjectiveTuning.R +++ b/R/ObjectiveTuning.R @@ -24,9 +24,6 @@ ObjectiveTuning = R6Class("ObjectiveTuning", #' @field learner ([mlr3::Learner]). learner = NULL, - #' @field resampling ([mlr3::Resampling]). - resampling = NULL, - #' @field measures (list of [mlr3::Measure]). measures = NULL, @@ -47,9 +44,7 @@ ObjectiveTuning = R6Class("ObjectiveTuning", self$task = assert_task(as_task(task, clone = TRUE)) self$learner = assert_learner(as_learner(learner, clone = TRUE)) - self$resampling = assert_resampling(as_resampling( - resampling, - clone = TRUE)) + self$resampling = assert_resampling(as_resampling(resampling, clone = TRUE)) self$measures = assert_measures(as_measures(measures, clone = TRUE), task = self$task, learner = self$learner) self$store_benchmark_result = assert_logical(store_benchmark_result) @@ -57,9 +52,6 @@ ObjectiveTuning = R6Class("ObjectiveTuning", if (self$store_models && !self$store_benchmark_result) { stop("Models can only be stored if store_benchmark_result is set to TRUE") } - if (!resampling$is_instantiated) { - self$resampling$instantiate(self$task) - } codomain = ParamSet$new(map(self$measures, function(s) { ParamDbl$new(id = s$id, tags = ifelse(s$minimize, "minimize", "maximize")) @@ -93,6 +85,22 @@ ObjectiveTuning = R6Class("ObjectiveTuning", } else { aggr[, y, with = FALSE] } + }, + + .resampling = NULL + ), + + active = list( + + #' @field resampling ([mlr3::Resampling]). + resampling = function(rhs) { + if(missing(rhs)) { + private$.resampling + } else { + resampling = assert_resampling(as_resampling(rhs, clone = TRUE)) + if (!resampling$is_instantiated) resampling$instantiate(self$task) + private$.resampling = resampling + } } ) ) From 0a98265ca5bea5f2f164265980d4b090292c9e21 Mon Sep 17 00:00:00 2001 From: Marc Becker Date: Thu, 14 Jan 2021 11:46:54 +0100 Subject: [PATCH 17/40] Rewrite Tuner --- R/TunerIrace.R | 38 +++++++++++++++++++++++++++++++------- R/irace_helpers.R | 26 +++++++++++++------------- 2 files changed, 44 insertions(+), 20 deletions(-) diff --git a/R/TunerIrace.R b/R/TunerIrace.R index 5dc6677d1..0cd301051 100644 --- a/R/TunerIrace.R +++ b/R/TunerIrace.R @@ -57,6 +57,7 @@ TunerIrace = R6Class("TunerIrace", #' Creates a new instance of this [R6][R6::R6Class] class. initialize = function() { ps = ParamSet$new(list( + ParamInt$new("n_instances", lower = 1, default = 10), ParamLgl$new("show.irace.output", default = FALSE), ParamInt$new("debugLevel", default = 0, lower = 0), ParamInt$new("seed"), @@ -85,7 +86,7 @@ TunerIrace = R6Class("TunerIrace", ParamDbl$new("boundAsTimeout", default = 1) )) - ps$values$show.irace.output = FALSE # nolint + ps$values = list(n_instances = 10, show.irace.output = FALSE) # nolint super$initialize( param_set = ps, @@ -98,12 +99,35 @@ TunerIrace = R6Class("TunerIrace", private = list( .optimize = function(inst) { - g = if (self$param_set$values$show.irace.output) identity else utils::capture.output - g(irace::irace( - scenario = c( - make_scenario(inst), - self$param_set$values[names(self$param_set$values) %nin% "show.irace.output"]), - parameters = paradox_to_irace(inst$search_space))) + pv = self$param_set$values + terminator = inst$terminator + objective = inst$objective + + # Hide irace output + g = if (pv$show.irace.output) identity else utils::capture.output + + # Set resampling instances + ri = map(seq(pv$n_instances), function(n) { + r = objective$resampling$clone() + r$instantiate(objective$task) + }) + + # Clean pv + pv$show.irace.output = NULL + pv$n_instances = NULL + + # Make scenario + scenario = c(list( + targetRunner = targetRunner, + logFile = tempfile(), + instances = ri, + debugLevel = 0, + maxExperiments = if (class(inst$terminator)[1] == "TerminatorEvals") terminator$param_set$values$n_evals else 0, + maxTime = if (class(inst$terminator)[1] == "TerminatorRunTime") terminator$param_set$values$secs else 0, + targetRunnerData = list(inst = inst) + ), pv) + + g(irace::irace(scenario = scenario, parameters = paradox_to_irace(inst$search_space))) } ) ) diff --git a/R/irace_helpers.R b/R/irace_helpers.R index 98b1e97a4..f8d3d0721 100644 --- a/R/irace_helpers.R +++ b/R/irace_helpers.R @@ -51,21 +51,12 @@ get_irace_condition = function(ps) { return(tab) } -make_scenario = function(inst) { - list( - targetRunner = targetRunner, - logFile = tempfile(), - instances = list(inst), - debugLevel = 0, - maxExperiments = if (class(inst$terminator)[1] == "TerminatorEvals") - inst$terminator$param_set$values$n_evals else 0, - maxTime = if (class(inst$terminator)[1] == "TerminatorRunTime") - inst$terminator$param_set$values$secs else 0 - ) -} + targetRunner = function(experiment, scenario) { # nolint t0 = Sys.time() + tuning_instance = scenario$targetRunnerData$inst + # fix logicals config = as.data.table(lapply(experiment$configuration, function(x) { if (x %in% c("TRUE", "FALSE")) { @@ -74,7 +65,16 @@ targetRunner = function(experiment, scenario) { # nolint return(x) } })) - cost = scenario$instances[[1]]$eval_batch(config)[[1]] + + # change resampling instance + tuning_instance$objective$resampling = experiment$instance + + # add extra info to archive + extra = data.table(id_configuration = experiment$id.configuration, id_instance = experiment$id.instance) + + # evaluate configuration + # objective_function cannot pass extra information + cost = as.numeric(tuning_instance$eval_batch(cbind(config, extra))) * tuning_instance$objective_multiplicator t1 = Sys.time() return(list(cost = cost, time = as.numeric(t1 - t0))) From e516889bc7ee771f151aa16e133774aa160598f3 Mon Sep 17 00:00:00 2001 From: Marc Becker Date: Thu, 14 Jan 2021 12:39:17 +0100 Subject: [PATCH 18/40] Fix code --- R/TunerIrace.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/TunerIrace.R b/R/TunerIrace.R index 0cd301051..951da2c44 100644 --- a/R/TunerIrace.R +++ b/R/TunerIrace.R @@ -105,7 +105,7 @@ TunerIrace = R6Class("TunerIrace", # Hide irace output g = if (pv$show.irace.output) identity else utils::capture.output - + # Set resampling instances ri = map(seq(pv$n_instances), function(n) { r = objective$resampling$clone() From 15a918f4e8467f9cb8d5dcfe6397b656fd1fdff9 Mon Sep 17 00:00:00 2001 From: Marc Becker Date: Thu, 14 Jan 2021 12:40:19 +0100 Subject: [PATCH 19/40] Fix code --- R/irace_helpers.R | 4 ---- 1 file changed, 4 deletions(-) diff --git a/R/irace_helpers.R b/R/irace_helpers.R index c112ef9d8..f8d3d0721 100644 --- a/R/irace_helpers.R +++ b/R/irace_helpers.R @@ -65,7 +65,6 @@ targetRunner = function(experiment, scenario) { # nolint return(x) } })) -<<<<<<< HEAD # change resampling instance tuning_instance$objective$resampling = experiment$instance @@ -76,9 +75,6 @@ targetRunner = function(experiment, scenario) { # nolint # evaluate configuration # objective_function cannot pass extra information cost = as.numeric(tuning_instance$eval_batch(cbind(config, extra))) * tuning_instance$objective_multiplicator -======= - cost = scenario$instances[[1]]$objective_function(config)#[[1]] ->>>>>>> f84cb78ee9688cb108609c82b0132ee14dd6f901 t1 = Sys.time() return(list(cost = cost, time = as.numeric(t1 - t0))) From 650f067d55af271df62ffc4c6deb46f245c9fd08 Mon Sep 17 00:00:00 2001 From: Marc Becker Date: Thu, 14 Jan 2021 13:04:18 +0100 Subject: [PATCH 20/40] Disable partial match warnings --- tests/testthat/setup.R | 13 +++++++------ tests/testthat/teardown.R | 2 +- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/tests/testthat/setup.R b/tests/testthat/setup.R index 8c9545a63..7deeb1b62 100644 --- a/tests/testthat/setup.R +++ b/tests/testthat/setup.R @@ -1,13 +1,14 @@ library(mlr3) attachNamespace("checkmate") -old_opts = options( - warnPartialMatchArgs = TRUE, - warnPartialMatchAttr = TRUE, - warnPartialMatchDollar = TRUE -) +# The upstream package causes partial match warnings +# old_opts = options( +# warnPartialMatchArgs = TRUE, +# warnPartialMatchAttr = TRUE, +# warnPartialMatchDollar = TRUE +# ) # https://github.com/HenrikBengtsson/Wishlist-for-R/issues/88 -old_opts = lapply(old_opts, function(x) if (is.null(x)) FALSE else x) +# old_opts = lapply(old_opts, function(x) if (is.null(x)) FALSE else x) lg_mlr3 = lgr::get_logger("mlr3") old_threshold_mlr3 = lg_mlr3$threshold diff --git a/tests/testthat/teardown.R b/tests/testthat/teardown.R index e72e77f51..f080252c7 100644 --- a/tests/testthat/teardown.R +++ b/tests/testthat/teardown.R @@ -1,3 +1,3 @@ -options(old_opts) +# options(old_opts) lg_mlr3$set_threshold(old_threshold_mlr3) lg_bbotk$set_threshold(old_threshold_bbotk) From be36da111fbf8f94c004d04b354e197c6e2a1031 Mon Sep 17 00:00:00 2001 From: Marc Becker Date: Thu, 14 Jan 2021 14:34:53 +0100 Subject: [PATCH 21/40] Add .assign_result --- R/TunerIrace.R | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/R/TunerIrace.R b/R/TunerIrace.R index 951da2c44..b233261b8 100644 --- a/R/TunerIrace.R +++ b/R/TunerIrace.R @@ -127,8 +127,24 @@ TunerIrace = R6Class("TunerIrace", targetRunnerData = list(inst = inst) ), pv) - g(irace::irace(scenario = scenario, parameters = paradox_to_irace(inst$search_space))) - } + g({res =irace::irace(scenario = scenario, parameters = paradox_to_irace(inst$search_space))}) + + # Temporarily store result + private$.result_id = res$.ID.[1] + }, + + # The final configurations returned by irace are the elites of the final race. + # We store the best performing one. + # The reported performance value is the average of all resampling iterations. + .assign_result = function(inst) { + res = inst$archive$data[id_configuration == private$.result_id, ] + cols = c(inst$archive$cols_x, "id_configuration") + xdt = res[1, cols, with = FALSE] + y = set_names(mean(unlist(res[, inst$archive$cols_y, with = FALSE])), inst$archive$cols_y) + inst$assign_result(xdt, y) + }, + + .result_id = NULL ) ) From 888395c039b3ac3d9a1dbd2d14074782a0ccbecb Mon Sep 17 00:00:00 2001 From: Marc Becker Date: Thu, 14 Jan 2021 14:51:27 +0100 Subject: [PATCH 22/40] Remove show.irace.output --- R/TunerIrace.R | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/R/TunerIrace.R b/R/TunerIrace.R index b233261b8..d8fb7b462 100644 --- a/R/TunerIrace.R +++ b/R/TunerIrace.R @@ -7,7 +7,7 @@ #' Subclass for iterated racing tuning calling [irace::irace()] from package \CRANpkg{irace}. #' #' @section Parameters: -#' * `show.irace.output` (`logical(1)`) +#' * `n_instances` (`interger(1)`) #' * `debugLevel` (`integer(1)`) #' * `seed` (`integer(1)`) #' * `postselection` (`numeric(1)`) @@ -33,7 +33,6 @@ #' * `boundPar` (`numeric(1)`) #' * `boundAsTimeout` (`numeric(1)`) #' -#' `show.irace.output` suppresses the output from [irace::irace()] and only prints output from this package. #' For the meaning of all other parameters, see [irace::defaultScenario()]. #' Note that we have removed all control parameters which refer to the termination of the algorithm and #' where our terminators allow to obtain the same behavior. @@ -58,7 +57,6 @@ TunerIrace = R6Class("TunerIrace", initialize = function() { ps = ParamSet$new(list( ParamInt$new("n_instances", lower = 1, default = 10), - ParamLgl$new("show.irace.output", default = FALSE), ParamInt$new("debugLevel", default = 0, lower = 0), ParamInt$new("seed"), ParamDbl$new("postselection", default = 0, lower = 0, upper = 1), @@ -86,7 +84,7 @@ TunerIrace = R6Class("TunerIrace", ParamDbl$new("boundAsTimeout", default = 1) )) - ps$values = list(n_instances = 10, show.irace.output = FALSE) # nolint + ps$values = list(n_instances = 10) # nolint super$initialize( param_set = ps, @@ -102,18 +100,12 @@ TunerIrace = R6Class("TunerIrace", pv = self$param_set$values terminator = inst$terminator objective = inst$objective - - # Hide irace output - g = if (pv$show.irace.output) identity else utils::capture.output # Set resampling instances ri = map(seq(pv$n_instances), function(n) { r = objective$resampling$clone() r$instantiate(objective$task) }) - - # Clean pv - pv$show.irace.output = NULL pv$n_instances = NULL # Make scenario @@ -127,7 +119,7 @@ TunerIrace = R6Class("TunerIrace", targetRunnerData = list(inst = inst) ), pv) - g({res =irace::irace(scenario = scenario, parameters = paradox_to_irace(inst$search_space))}) + res = irace::irace(scenario = scenario, parameters = paradox_to_irace(inst$search_space)) # Temporarily store result private$.result_id = res$.ID.[1] From f929832af4665f4958df738e9d8261c578faf28f Mon Sep 17 00:00:00 2001 From: Marc Becker Date: Thu, 14 Jan 2021 16:48:25 +0100 Subject: [PATCH 23/40] Fix unit tests --- R/TunerIrace.R | 4 +- R/irace_helpers.R | 10 +++-- tests/testthat.R | 1 + tests/testthat/test_TunerIrace.R | 39 ++++--------------- .../testthat/test_TuningInstanceSingleCrit.R | 4 +- 5 files changed, 19 insertions(+), 39 deletions(-) diff --git a/R/TunerIrace.R b/R/TunerIrace.R index d8fb7b462..cd91a28c2 100644 --- a/R/TunerIrace.R +++ b/R/TunerIrace.R @@ -71,7 +71,7 @@ TunerIrace = R6Class("TunerIrace", ParamDbl$new("softRestartThreshold"), ParamInt$new("digits", default = 4, lower = 1, upper = 15), ParamFct$new("testType", default = "F-test", - levels = c("F-test", "t-test", "t-test-bonferroni", "t-test-holm")), + levels = c("F-test", "t-test", "t-test-bonferroni", "t-test-holm")), ParamInt$new("firstTest", default = 5, lower = 0), ParamInt$new("eachTest", default = 1, lower = 1), ParamDbl$new("confidence", default = 0.95, lower = 0, upper = 1), @@ -120,7 +120,7 @@ TunerIrace = R6Class("TunerIrace", ), pv) res = irace::irace(scenario = scenario, parameters = paradox_to_irace(inst$search_space)) - + # Temporarily store result private$.result_id = res$.ID.[1] }, diff --git a/R/irace_helpers.R b/R/irace_helpers.R index f8d3d0721..f4e689cee 100644 --- a/R/irace_helpers.R +++ b/R/irace_helpers.R @@ -1,13 +1,16 @@ paradox_to_irace = function(ps) { assertClass(ps, "ParamSet") + if("ParamUty" %in% ps$class) { + stop(" not supported by ") + } + # what about ParamUty = vector numeric/real class_lookup = data.table( paradox = c("ParamLgl", "ParamInt", "ParamDbl", "ParamFct"), irace = c("c", "i", "r", "c"), stringsAsFactors = FALSE) - type = unlist(subset(merge(data.table(paradox = ps$class), class_lookup, sort = FALSE), - select = "irace")) + type = unlist(subset(merge(data.table(paradox = ps$class), class_lookup, sort = FALSE), select = "irace")) range = get_irace_range(ps) if (ps$has_deps) { condition = get_irace_condition(ps) @@ -19,6 +22,7 @@ paradox_to_irace = function(ps) { return(irace::readParameters(text = par_tab)) } + get_irace_range = function(ps) { rng = data.table(lower = ps$lower, upper = ps$upper, lvl = ps$levels) @@ -30,6 +34,7 @@ get_irace_range = function(ps) { } }) } + get_irace_condition = function(ps) { cond = rbindlist(apply(ps$deps, 1, function(x) { on = x[[2]] @@ -52,7 +57,6 @@ get_irace_condition = function(ps) { return(tab) } - targetRunner = function(experiment, scenario) { # nolint t0 = Sys.time() tuning_instance = scenario$targetRunnerData$inst diff --git a/tests/testthat.R b/tests/testthat.R index ab592c5ff..930a8b84b 100644 --- a/tests/testthat.R +++ b/tests/testthat.R @@ -3,6 +3,7 @@ if (requireNamespace("testthat", quietly = TRUE)) { library(checkmate) library(mlr3) library(mlr3tuning) + library(mlr3learners) test_check("mlr3tuning") } diff --git a/tests/testthat/test_TunerIrace.R b/tests/testthat/test_TunerIrace.R index 18aff7488..b571c3c83 100644 --- a/tests/testthat/test_TunerIrace.R +++ b/tests/testthat/test_TunerIrace.R @@ -1,4 +1,3 @@ -library(mlr3learners) set.seed(1) skip_if_not_installed("irace") @@ -20,8 +19,7 @@ test_that("TunerIrace with int params and trafo, clock terminator", { tsk("iris"), lrn("classif.rpart"), rsmp("holdout"), msr("classif.ce"), trm("run_time", secs = 24), ps) tt = tnr("irace") - # suppressWarnings used throughout - looks like coming upstream - suppressWarnings(tt$optimize(inst)) + tt$optimize(inst) expect_double(inst$archive$best()$cp) }) @@ -36,7 +34,7 @@ test_that("TunerIrace with dependencies", { rsmp("holdout"), msr("classif.ce"), trm("evals", n_evals = 96), ps) tt = tnr("irace") - suppressWarnings(tt$optimize(inst)) + tt$optimize(inst) expect_double(inst$archive$best()$cp) }) @@ -48,7 +46,7 @@ test_that("minimize time", { tsk("iris"), lrn("classif.rpart"), rsmp("holdout"), msr("classif.ce"), trm("run_time", secs = 20), ps) tt = tnr("irace", capping = 1, boundMax = 1, cappingType = "best", boundType = "instance") - suppressWarnings(tt$optimize(inst)) + tt$optimize(inst) expect_double(inst$archive$best()$cp) }) @@ -63,7 +61,7 @@ test_that("paradox_to_irace no dependencies", { ps = ParamSet$new(params = list( ParamUty$new("uty") )) - expect_error(paradox_to_irace(ps)) + expect_error(paradox_to_irace(ps), regex = " not supported by ", fixed = TRUE) ps = ParamSet$new(params = list( ParamDbl$new("dbl", lower = 0.1, upper = 0.3), @@ -126,7 +124,6 @@ test_that("paradox_to_irace dependencies", { depends = list(a = character(0), c = "b", b = "a"), hierarchy = c(1, 3, 2)) - ps = ParamSet$new(params = list( ParamLgl$new("a"), ParamInt$new("b", lower = 1, upper = 9), @@ -159,7 +156,7 @@ test_that("TunerIrace works with logical params", { msr("regr.mse"), trm("evals", n_evals = 42), ps) tt = tnr("irace") - suppressWarnings(tt$optimize(inst)) + tt$optimize(inst) expect_logical(inst$archive$best()$keep_model) }) @@ -172,7 +169,7 @@ test_that("TunerIrace works with tune.threshold", { tsk("iris"), lrn("classif.rpart"), rsmp("holdout", ratio = 0.1), msr("classif.ce"), trm("evals", n_evals = 50), ps) - suppressWarnings(tt$optimize(inst)) + tt$optimize(inst) expect_double(inst$archive$best()$minsplit) }) @@ -184,29 +181,7 @@ test_that("TunerIrace uses digits", { inst = TuningInstanceSingleCrit$new( tsk("iris"), lrn("classif.rpart"), rsmp("holdout"), msr("classif.ce"), trm("evals", n_evals = 30), ps) - suppressWarnings(tt$optimize(inst)) - expect_double(inst$archive$best()$cp) - - - ps = ParamSet$new(params = list( - ParamDbl$new("cp", lower = 1e-5, upper = 1e-4) - )) - tt = tnr("irace", digits = 3L) - inst = TuningInstanceSingleCrit$new( - tsk("iris"), lrn("classif.rpart"), rsmp("holdout"), - msr("classif.ce"), trm("evals", n_evals = 40), ps) - expect_error(suppressWarnings(tt$optimize(inst))) -}) - -test_that("Error in hyperparameter tuning with scientific notation for lower/upper boundaries", { - ps = ParamSet$new(params = list( - ParamDbl$new("cp", lower = 1e-3, upper = 1e-1) - )) - inst = TuningInstanceSingleCrit$new( - tsk("iris"), lrn("classif.rpart"), rsmp("holdout"), - msr("classif.ce"), trm("evals", n_evals = 30), ps) - tt = tnr("irace", nbIterations = 1L) - suppressWarnings(tt$optimize(inst)) + tt$optimize(inst) expect_double(inst$archive$best()$cp) }) diff --git a/tests/testthat/test_TuningInstanceSingleCrit.R b/tests/testthat/test_TuningInstanceSingleCrit.R index eee342c66..0140b34f4 100644 --- a/tests/testthat/test_TuningInstanceSingleCrit.R +++ b/tests/testthat/test_TuningInstanceSingleCrit.R @@ -17,7 +17,7 @@ test_that("TuningInstanceSingleCrit", { expect_equal(inst$archive$benchmark_result$resample_result(2)$learners[[1]]$param_set$values$maxdepth, 10) expect_identical(inst$archive$n_evals, 2L) expect_data_table(z, nrows = 2) - expect_named(z, c("dummy.cp.classif", "uhash")) + expect_named(z, c("dummy.cp.classif")) z = inst$eval_batch(data.table(cp = c(0.2, 0.1), minsplit = c(3, 4))) expect_data_table(inst$archive$data, nrows = 4L) @@ -29,7 +29,7 @@ test_that("TuningInstanceSingleCrit", { expect_equal(inst$archive$benchmark_result$resample_result(4)$learners[[1]]$param_set$values$maxdepth, 10) expect_identical(inst$archive$n_evals, 4L) expect_data_table(z, nrows = 2L) - expect_named(z, c("dummy.cp.classif", "uhash")) + expect_named(z, c("dummy.cp.classif")) # test archive a = inst$archive$data From 33bfdae8caa02b4c3d8fb60bbf698fee4fa9ce6f Mon Sep 17 00:00:00 2001 From: Marc Becker Date: Thu, 14 Jan 2021 16:55:38 +0100 Subject: [PATCH 24/40] Fix style --- R/TunerIrace.R | 1 + tests/testthat/test_TuningInstanceSingleCrit.R | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/R/TunerIrace.R b/R/TunerIrace.R index cd91a28c2..b3a603811 100644 --- a/R/TunerIrace.R +++ b/R/TunerIrace.R @@ -129,6 +129,7 @@ TunerIrace = R6Class("TunerIrace", # We store the best performing one. # The reported performance value is the average of all resampling iterations. .assign_result = function(inst) { + id_configuration = NULL res = inst$archive$data[id_configuration == private$.result_id, ] cols = c(inst$archive$cols_x, "id_configuration") xdt = res[1, cols, with = FALSE] diff --git a/tests/testthat/test_TuningInstanceSingleCrit.R b/tests/testthat/test_TuningInstanceSingleCrit.R index 0140b34f4..9142b0fd1 100644 --- a/tests/testthat/test_TuningInstanceSingleCrit.R +++ b/tests/testthat/test_TuningInstanceSingleCrit.R @@ -17,7 +17,7 @@ test_that("TuningInstanceSingleCrit", { expect_equal(inst$archive$benchmark_result$resample_result(2)$learners[[1]]$param_set$values$maxdepth, 10) expect_identical(inst$archive$n_evals, 2L) expect_data_table(z, nrows = 2) - expect_named(z, c("dummy.cp.classif")) + expect_named(z, "dummy.cp.classif") z = inst$eval_batch(data.table(cp = c(0.2, 0.1), minsplit = c(3, 4))) expect_data_table(inst$archive$data, nrows = 4L) @@ -29,7 +29,7 @@ test_that("TuningInstanceSingleCrit", { expect_equal(inst$archive$benchmark_result$resample_result(4)$learners[[1]]$param_set$values$maxdepth, 10) expect_identical(inst$archive$n_evals, 4L) expect_data_table(z, nrows = 2L) - expect_named(z, c("dummy.cp.classif")) + expect_named(z, "dummy.cp.classif") # test archive a = inst$archive$data From 1fc81b8e0c1f029c2b7446e05c3729ecad839634 Mon Sep 17 00:00:00 2001 From: Marc Becker Date: Fri, 15 Jan 2021 09:20:53 +0100 Subject: [PATCH 25/40] Add error message if no result is found --- R/TunerIrace.R | 3 +++ 1 file changed, 3 insertions(+) diff --git a/R/TunerIrace.R b/R/TunerIrace.R index b3a603811..f54d23cb8 100644 --- a/R/TunerIrace.R +++ b/R/TunerIrace.R @@ -129,6 +129,9 @@ TunerIrace = R6Class("TunerIrace", # We store the best performing one. # The reported performance value is the average of all resampling iterations. .assign_result = function(inst) { + if(length(private$.result_id) == 0) { + stop("irace::irace did not return a result. The evaluated configurations are still accessible through the archive.") + } id_configuration = NULL res = inst$archive$data[id_configuration == private$.result_id, ] cols = c(inst$archive$cols_x, "id_configuration") From 7096ba511d3ab6c1a92e8b8c927520a991f58ad9 Mon Sep 17 00:00:00 2001 From: Marc Becker Date: Fri, 15 Jan 2021 11:33:12 +0100 Subject: [PATCH 26/40] Update unit tests --- tests/testthat/test_TunerIrace.R | 235 ++++++++++++------------------- 1 file changed, 93 insertions(+), 142 deletions(-) diff --git a/tests/testthat/test_TunerIrace.R b/tests/testthat/test_TunerIrace.R index b571c3c83..a8fd87536 100644 --- a/tests/testthat/test_TunerIrace.R +++ b/tests/testthat/test_TunerIrace.R @@ -1,121 +1,99 @@ -set.seed(1) - skip_if_not_installed("irace") test_that("TunerIrace", { - test_tuner("irace", term_evals = 42, real_evals = 39) -}) + z = test_tuner("irace", term_evals = 42, real_evals = 39) + a = z$inst$archive$data + expect_subset(c("id_configuration", "id_instance"), names(a)) -test_that("TunerIrace with int params and trafo, clock terminator", { - ps = ParamSet$new(params = list( - ParamDbl$new("cp", lower = 0.001, upper = 0.1), - ParamDbl$new("minsplit", lower = 1, upper = 10) - )) - ps$trafo = function(x, param_set) { - x$minsplit = as.integer(round(x$minsplit)) - return(x) - } - inst = TuningInstanceSingleCrit$new( - tsk("iris"), lrn("classif.rpart"), rsmp("holdout"), - msr("classif.ce"), trm("run_time", secs = 24), ps) - tt = tnr("irace") - tt$optimize(inst) - expect_double(inst$archive$best()$cp) + # check for mean performance + expect_equal(z$inst$result$classif.ce, mean(a[id_configuration == z$inst$result$id_configuration,]$classif.ce)) }) -test_that("TunerIrace with dependencies", { - ps = ParamSet$new(params = list( - ParamDbl$new("cp", lower = 0.001, upper = 0.1), - ParamInt$new("minsplit", lower = 1, upper = 10) - )) - ps$add_dep("minsplit", "cp", CondEqual$new(0.005)) - inst = TuningInstanceSingleCrit$new( - tsk("iris"), lrn("classif.rpart"), - rsmp("holdout"), msr("classif.ce"), - trm("evals", n_evals = 96), ps) - tt = tnr("irace") - tt$optimize(inst) - expect_double(inst$archive$best()$cp) +test_that("TunerIrace works with TerminatorRunTime", { + search_space = ps( + cp = p_dbl(lower = 0.001, upper = 0.1), + minsplit = p_int(lower = 1, upper = 10) + ) + instance = TuningInstanceSingleCrit$new(tsk("iris"), lrn("classif.rpart"), rsmp("holdout"), msr("classif.ce"), + trm("run_time", secs = 30), search_space) + tuner = tnr("irace") + expect_data_table(tuner$optimize(instance)) }) -test_that("minimize time", { - ps = ParamSet$new(params = list( - ParamDbl$new("cp", lower = 0.001, upper = 0.1) - )) - inst = TuningInstanceSingleCrit$new( - tsk("iris"), lrn("classif.rpart"), rsmp("holdout"), - msr("classif.ce"), trm("run_time", secs = 20), ps) - tt = tnr("irace", capping = 1, boundMax = 1, cappingType = "best", boundType = "instance") - tt$optimize(inst) - expect_double(inst$archive$best()$cp) +test_that("TunerIrace works with dependencies", { + search_space = ps( + cp = p_dbl(lower = 0.001, upper = 0.1), + minsplit = p_int(lower = 1, upper = 10, depends = cp == 0.005) + ) + instance = TuningInstanceSingleCrit$new(tsk("iris"), lrn("classif.rpart"), rsmp("holdout"), msr("classif.ce"), + trm("evals", n_evals = 96), search_space) + tuner = tnr("irace") + tuner$optimize(instance) + a = instance$archive$data + expect_true(all(is.na(a[cp != 0.005, minsplit]))) + expect_double(a$cp) }) -test_that("paradox_to_irace no dependencies", { - ps = ParamSet$new(params = list( - ParamLgl$new("lgl") - )) - expect_irace_parameters( - parameters = paradox_to_irace(ps), names = "lgl", types = "c", +test_that("paradox_to_irace without dependencies", { + # only ParamLgl + pps = ps(lgl = p_lgl()) + expect_irace_parameters(parameters = paradox_to_irace(pps), names = "lgl", types = "c", domain = list(lgl = c("TRUE", "FALSE")), conditions = list(lgl = TRUE)) - - ps = ParamSet$new(params = list( - ParamUty$new("uty") - )) - expect_error(paradox_to_irace(ps), regex = " not supported by ", fixed = TRUE) - - ps = ParamSet$new(params = list( - ParamDbl$new("dbl", lower = 0.1, upper = 0.3), - ParamInt$new("int", lower = 1, upper = 9), - ParamFct$new("fct", levels = c("a", "b", "c")), - ParamLgl$new("lgl") - )) + + # only ParamUty + pps = ps(uty = p_uty()) + expect_error(paradox_to_irace(pps), regex = " not supported by ", fixed = TRUE) + + # mixed set + pps = ps( + dbl = p_dbl(lower = 0.1, upper = 0.3), + int = p_int(lower = 1, upper = 9), + fct = p_fct(levels = c("a", "b", "c")), + lgl = p_lgl() + ) expect_irace_parameters( - parameters = paradox_to_irace(ps), + parameters = paradox_to_irace(pps), names = c("dbl", "int", "fct", "lgl"), types = c("r", "i", "c", "c"), - domain = list( - dbl = c(0.1, 0.3), int = c(1, 9), fct = c("a", "b", "c"), - lgl = c("TRUE", "FALSE"))) + domain = list(dbl = c(0.1, 0.3), int = c(1, 9), fct = c("a", "b", "c"), lgl = c("TRUE", "FALSE"))) # double checking previous bug in merge sort - ps = ParamSet$new(params = list( - ParamFct$new("fct", levels = c("a", "b", "c")), - ParamInt$new("int1", lower = 1, upper = 9), - ParamDbl$new("dbl", lower = 0.1, upper = 0.3), - ParamInt$new("int2", lower = 10, upper = 90), - ParamLgl$new("lgl") - )) + pps = ps( + fct = p_fct(levels = c("a", "b", "c")), + int1 = p_int(lower = 1, upper = 9), + dbl = p_dbl(lower = 0.1, upper = 0.3), + int2 = p_int(lower = 10, upper = 90), + lgl = p_lgl() + ) expect_irace_parameters( - parameters = paradox_to_irace(ps), + parameters = paradox_to_irace(pps), names = c("fct", "int1", "dbl", "int2", "lgl"), types = c("c", "i", "r", "i", "c"), - domain = list( - fct = c("a", "b", "c"), int1 = c(1, 9), dbl = c(0.1, 0.3), - int2 = c(10, 90), lgl = c("TRUE", "FALSE"))) + domain = list(fct = c("a", "b", "c"), int1 = c(1, 9), dbl = c(0.1, 0.3), int2 = c(10, 90), + lgl = c("TRUE", "FALSE"))) }) -test_that("paradox_to_irace dependencies", { - ps = ParamSet$new(params = list( - ParamLgl$new("a"), - ParamInt$new("b", lower = 1, upper = 9)) +test_that("paradox_to_irace with dependencies", { + # one dependency + pps = ps( + a = p_lgl(), + b = p_int(lower = 1, upper = 9, depends = a == TRUE) ) - ps$add_dep("b", "a", CondEqual$new(TRUE)) expect_irace_parameters( - parameters = paradox_to_irace(ps), names = c("a", "b"), types = c("c", "i"), + parameters = paradox_to_irace(pps), names = c("a", "b"), types = c("c", "i"), domain = list(a = c("TRUE", "FALSE"), b = c(1, 9)), conditions = list(a = TRUE, b = expression(a == TRUE)), depends = list(a = character(0), b = "a"), hierarchy = c(1, 2)) - ps = ParamSet$new(params = list( - ParamLgl$new("a"), - ParamFct$new("c", levels = c("lvl1", "lvl2")), - ParamInt$new("b", lower = 1, upper = 9) - )) - ps$add_dep("b", "a", CondEqual$new(TRUE)) - ps$add_dep("c", "b", CondAnyOf$new(c(2, 5, 7))) + # two dependencies + pps = ps( + a = p_lgl(), + c = p_fct(levels = c("lvl1", "lvl2"), depends = b %in% c(2, 5, 7)), + b = p_int(lower = 1, upper = 9, depends = a == TRUE) + ) expect_irace_parameters( - parameters = paradox_to_irace(ps), names = c("a", "c", "b"), + parameters = paradox_to_irace(pps), names = c("a", "c", "b"), types = c("c", "c", "i"), domain = list(a = c("TRUE", "FALSE"), c = c("lvl1", "lvl2"), b = c(1, 9)), conditions = list( @@ -124,17 +102,15 @@ test_that("paradox_to_irace dependencies", { depends = list(a = character(0), c = "b", b = "a"), hierarchy = c(1, 3, 2)) - ps = ParamSet$new(params = list( - ParamLgl$new("a"), - ParamInt$new("b", lower = 1, upper = 9), - ParamFct$new("c", levels = c("lvl1", "lvl2")), - ParamDbl$new("d", lower = 0, upper = 1) - )) - ps$add_dep("b", "a", CondEqual$new(TRUE)) - ps$add_dep("a", "c", CondEqual$new("lvl1")) - ps$add_dep("d", "c", CondAnyOf$new(c("lvl1", "lvl2"))) + # three dependencies + pps = ps( + a = p_lgl(depends = c == "lvl1"), + b = p_int(lower = 1, upper = 9, depends = a == TRUE), + c = p_fct(levels = c("lvl1", "lvl2")), + d = p_dbl(lower = 0, upper = 1, depends = c %in% c("lvl1", "lvl2")) + ) expect_irace_parameters( - parameters = paradox_to_irace(ps), names = c("a", "b", "c", "d"), + parameters = paradox_to_irace(pps), names = c("a", "b", "c", "d"), types = c("c", "i", "c", "r"), domain = list( a = c("TRUE", "FALSE"), b = c(1, 9), c = c("lvl1", "lvl2"), @@ -147,53 +123,28 @@ test_that("paradox_to_irace dependencies", { hierarchy = c(2, 3, 1, 2)) }) -test_that("TunerIrace works with logical params", { - ps = ParamSet$new(params = list( - ParamLgl$new("keep_model") - )) - inst = TuningInstanceSingleCrit$new( - tsk("mtcars"), lrn("regr.rpart"), rsmp("holdout"), - msr("regr.mse"), - trm("evals", n_evals = 42), ps) - tt = tnr("irace") - tt$optimize(inst) - expect_logical(inst$archive$best()$keep_model) -}) - -test_that("TunerIrace works with tune.threshold", { - ps = ParamSet$new(params = list( - ParamInt$new("minsplit", lower = 1, upper = 3) - )) - tt = tnr("irace", nbIterations = 1L, minNbSurvival = 1) - inst = TuningInstanceSingleCrit$new( - tsk("iris"), lrn("classif.rpart"), - rsmp("holdout", ratio = 0.1), msr("classif.ce"), - trm("evals", n_evals = 50), ps) - tt$optimize(inst) - expect_double(inst$archive$best()$minsplit) +test_that("TunerIrace works with logical parameters", { + pps = ps(keep_model = p_lgl()) + instance = TuningInstanceSingleCrit$new(tsk("mtcars"), lrn("regr.rpart"), rsmp("holdout"), msr("regr.mse"), + trm("evals", n_evals = 42), pps) + tuner = tnr("irace") + tuner$optimize(instance) + expect_logical(instance$archive$best()$keep_model) }) test_that("TunerIrace uses digits", { - tt = tnr("irace", nbIterations = 1L, minNbSurvival = 1) - ps = ParamSet$new(params = list( - ParamDbl$new("cp", lower = pi * 1e-20, upper = 5.242e12 / 1e13) - )) - inst = TuningInstanceSingleCrit$new( - tsk("iris"), lrn("classif.rpart"), rsmp("holdout"), - msr("classif.ce"), trm("evals", n_evals = 30), ps) - tt$optimize(inst) - expect_double(inst$archive$best()$cp) + pps = ps(cp = p_dbl(lower = pi * 1e-20, upper = 5.242e12 / 1e13)) + instance = TuningInstanceSingleCrit$new(tsk("iris"), lrn("classif.rpart"), rsmp("holdout"), msr("classif.ce"), + trm("evals", n_evals = 30), pps) + tuner = tnr("irace", nbIterations = 1L, minNbSurvival = 1) + expect_data_table(tuner$optimize(instance)) }) -# we had a bug here, see (mlr) issue #627 test_that("irace works with unnamed discrete values", { - ps = ParamSet$new(params = list( - ParamInt$new("minsplit", lower = 2L, upper = 7L) - )) - inst = TuningInstanceSingleCrit$new( - tsk("iris"), lrn("classif.rpart"), rsmp("holdout"), msr("classif.ce"), - trm("evals", n_evals = 50), ps) - tt = tnr("irace") - suppressWarnings(tt$optimize(inst)) - expect_double(inst$archive$best()$minsplit) + # we had a bug here, see (mlr) issue #627 + pps = ps(minsplit = p_int(lower = 2L, upper = 7L)) + inst = TuningInstanceSingleCrit$new(tsk("iris"), lrn("classif.rpart"), rsmp("holdout"), msr("classif.ce"), + trm("evals", n_evals = 50), pps) + tuner = tnr("irace") + expect_data_table(tuner$optimize(inst)) }) From 758d84b2f3de93ce8e816ea42288b9d03e35c0ff Mon Sep 17 00:00:00 2001 From: Marc Becker Date: Fri, 15 Jan 2021 12:01:10 +0100 Subject: [PATCH 27/40] Update doc and example --- NEWS.md | 4 +- R/TunerIrace.R | 66 +++++++++++++++++---------------- man/ObjectiveTuning.Rd | 9 ++++- man/mlr_tuners_cmaes.Rd | 4 +- man/mlr_tuners_design_points.Rd | 4 +- man/mlr_tuners_gensa.Rd | 4 +- man/mlr_tuners_grid_search.Rd | 4 +- man/mlr_tuners_irace.Rd | 63 +++++++++++++++---------------- man/mlr_tuners_nloptr.Rd | 4 +- man/mlr_tuners_random_search.Rd | 4 +- 10 files changed, 94 insertions(+), 72 deletions(-) diff --git a/NEWS.md b/NEWS.md index b4901eabf..1eeaa33fe 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,10 +1,10 @@ # mlr3tuning 0.5.0.9000 -- Internal changes only. +- Adds `TunerIrace` from `irace` package. # mlr3tuning 0.5.0 -- Adds `TunerCmaes` from adagio package. +- Adds `TunerCmaes` from `adagio` package. - Fix `predict_type` in `AutoTuner`. - Support to set `TuneToken` in `Learner$param_set` and create a search space from it. diff --git a/R/TunerIrace.R b/R/TunerIrace.R index f54d23cb8..6a5862afa 100644 --- a/R/TunerIrace.R +++ b/R/TunerIrace.R @@ -4,38 +4,19 @@ #' @include Tuner.R #' #' @description -#' Subclass for iterated racing tuning calling [irace::irace()] from package \CRANpkg{irace}. +#' `TunerIrace` class that implements iterated racing. Calls [irace::irace()] +#' from package \CRANpkg{irace}. #' #' @section Parameters: -#' * `n_instances` (`interger(1)`) -#' * `debugLevel` (`integer(1)`) -#' * `seed` (`integer(1)`) -#' * `postselection` (`numeric(1)`) -#' * `elitist` (`integer(1)`) -#' * `elitistLimit` (`integer(1)`) -#' * `nbIterations` (`integer(1)`) -#' * `nbExperimentsPerIteration` (`integer(1)`) -#' * `minNbSurvival` (`integer(1)`) -#' * `nbConfigurations` (`integer(1)`) -#' * `mu` (`integer(1)`) -#' * `softRestart` (`integer(1)`) -#' * `softRestartThreshold` (`numeric(1)`) -#' * `digits` (`integer(1)`) -#' * `testType` (`character(1)`) -#' * `firstTest` (`integer(1)`) -#' * `eachTest` (`integer(1)`) -#' * `confidence` (`numeric(1)`) -#' * `capping` (`integer(1)`) -#' * `cappingType` (`character(1)`) -#' * `boundType` (`character(1)`) -#' * `boundMax` (`numeric(1)`) -#' * `boundDigits` (`integer(1)`) -#' * `boundPar` (`numeric(1)`) -#' * `boundAsTimeout` (`numeric(1)`) +#' \describe{ +#' \item{`n_instances`}{`integer(1)`\cr +#' Number of resampling instances.} +#' } #' #' For the meaning of all other parameters, see [irace::defaultScenario()]. -#' Note that we have removed all control parameters which refer to the termination of the algorithm and -#' where our terminators allow to obtain the same behavior. +#' Note that we have removed all control parameters which refer to the +#' termination of the algorithm. Use [TerminatorRunTime] or [TerminatorEvals] +#' instead. Other terminators do not work with `TunerIrace`. #' #' @templateVar id irace #' @template section_dictionary_tuners @@ -48,7 +29,31 @@ #' @family Tuner #' @export #' @examples -#' # see ?Tuner +#' library(mlr3) +#' library(paradox) +#' search_space = ParamSet$new(list( +#' ParamDbl$new("cp", lower = 0.001, upper = 0.1) +#' )) +#' terminator = trm("evals", n_evals = 42) +#' instance = TuningInstanceSingleCrit$new( +#' task = tsk("iris"), +#' learner = lrn("classif.rpart"), +#' resampling = rsmp("holdout"), +#' measure = msr("classif.ce"), +#' search_space = search_space, +#' terminator = terminator +#' ) +#' tt = tnr("irace") +#' +#' # modifies the instance by reference +#' tt$optimize(instance) +#' +#' # returns best configuration and best performance +#' instance$result +#' +#' # allows access of data.table of full path of all evaluations +#' instance$archive + TunerIrace = R6Class("TunerIrace", inherit = Tuner, public = list( @@ -83,8 +88,7 @@ TunerIrace = R6Class("TunerIrace", ParamDbl$new("boundPar", default = 1), ParamDbl$new("boundAsTimeout", default = 1) )) - - ps$values = list(n_instances = 10) # nolint + ps$values = list(n_instances = 10) super$initialize( param_set = ps, diff --git a/man/ObjectiveTuning.Rd b/man/ObjectiveTuning.Rd index 91179b1c9..0b0cc5ce5 100644 --- a/man/ObjectiveTuning.Rd +++ b/man/ObjectiveTuning.Rd @@ -18,8 +18,6 @@ by the \link{TuningInstanceSingleCrit} / \link{TuningInstanceMultiCrit}. \item{\code{learner}}{(\link[mlr3:Learner]{mlr3::Learner}).} -\item{\code{resampling}}{(\link[mlr3:Resampling]{mlr3::Resampling}).} - \item{\code{measures}}{(list of \link[mlr3:Measure]{mlr3::Measure}).} \item{\code{store_models}}{(\code{logical(1)}).} @@ -30,6 +28,13 @@ by the \link{TuningInstanceSingleCrit} / \link{TuningInstanceMultiCrit}. } \if{html}{\out{}} } +\section{Active bindings}{ +\if{html}{\out{
}} +\describe{ +\item{\code{resampling}}{(\link[mlr3:Resampling]{mlr3::Resampling}).} +} +\if{html}{\out{
}} +} \section{Methods}{ \subsection{Public methods}{ \itemize{ diff --git a/man/mlr_tuners_cmaes.Rd b/man/mlr_tuners_cmaes.Rd index c690bae94..8727ee289 100644 --- a/man/mlr_tuners_cmaes.Rd +++ b/man/mlr_tuners_cmaes.Rd @@ -11,7 +11,8 @@ from package \CRANpkg{adagio}. \section{Dictionary}{ This \link{Tuner} can be instantiated via the \link[mlr3misc:Dictionary]{dictionary} -\link{mlr_tuners} or with the associated sugar function \code{\link[=tnr]{tnr()}}:\preformatted{mlr_tuners$get("cmaes") +\link{mlr_tuners} or with the associated sugar function \code{\link[=tnr]{tnr()}}:\preformatted{TunerCmaes$new() +mlr_tuners$get("cmaes") tnr("cmaes") } } @@ -71,6 +72,7 @@ Other Tuner: \code{\link{mlr_tuners_design_points}}, \code{\link{mlr_tuners_gensa}}, \code{\link{mlr_tuners_grid_search}}, +\code{\link{mlr_tuners_irace}}, \code{\link{mlr_tuners_nloptr}}, \code{\link{mlr_tuners_random_search}} } diff --git a/man/mlr_tuners_design_points.Rd b/man/mlr_tuners_design_points.Rd index eb4a06547..6bfb75d8f 100644 --- a/man/mlr_tuners_design_points.Rd +++ b/man/mlr_tuners_design_points.Rd @@ -13,7 +13,8 @@ in the design are evaluated in order as given. \section{Dictionary}{ This \link{Tuner} can be instantiated via the \link[mlr3misc:Dictionary]{dictionary} -\link{mlr_tuners} or with the associated sugar function \code{\link[=tnr]{tnr()}}:\preformatted{mlr_tuners$get("design_points") +\link{mlr_tuners} or with the associated sugar function \code{\link[=tnr]{tnr()}}:\preformatted{TunerDesignPoints$new() +mlr_tuners$get("design_points") tnr("design_points") } } @@ -83,6 +84,7 @@ Other Tuner: \code{\link{mlr_tuners_cmaes}}, \code{\link{mlr_tuners_gensa}}, \code{\link{mlr_tuners_grid_search}}, +\code{\link{mlr_tuners_irace}}, \code{\link{mlr_tuners_nloptr}}, \code{\link{mlr_tuners_random_search}} } diff --git a/man/mlr_tuners_gensa.Rd b/man/mlr_tuners_gensa.Rd index 917989a31..dcdabf58a 100644 --- a/man/mlr_tuners_gensa.Rd +++ b/man/mlr_tuners_gensa.Rd @@ -22,7 +22,8 @@ from package \CRANpkg{GenSA}. \section{Dictionary}{ This \link{Tuner} can be instantiated via the \link[mlr3misc:Dictionary]{dictionary} -\link{mlr_tuners} or with the associated sugar function \code{\link[=tnr]{tnr()}}:\preformatted{mlr_tuners$get("gensa") +\link{mlr_tuners} or with the associated sugar function \code{\link[=tnr]{tnr()}}:\preformatted{TunerGenSA$new() +mlr_tuners$get("gensa") tnr("gensa") } } @@ -85,6 +86,7 @@ Other Tuner: \code{\link{mlr_tuners_cmaes}}, \code{\link{mlr_tuners_design_points}}, \code{\link{mlr_tuners_grid_search}}, +\code{\link{mlr_tuners_irace}}, \code{\link{mlr_tuners_nloptr}}, \code{\link{mlr_tuners_random_search}} } diff --git a/man/mlr_tuners_grid_search.Rd b/man/mlr_tuners_grid_search.Rd index be490527b..25529f39a 100644 --- a/man/mlr_tuners_grid_search.Rd +++ b/man/mlr_tuners_grid_search.Rd @@ -14,7 +14,8 @@ evaluated in a random order. \section{Dictionary}{ This \link{Tuner} can be instantiated via the \link[mlr3misc:Dictionary]{dictionary} -\link{mlr_tuners} or with the associated sugar function \code{\link[=tnr]{tnr()}}:\preformatted{mlr_tuners$get("grid_search") +\link{mlr_tuners} or with the associated sugar function \code{\link[=tnr]{tnr()}}:\preformatted{TunerGridSearch$new() +mlr_tuners$get("grid_search") tnr("grid_search") } } @@ -88,6 +89,7 @@ Other Tuner: \code{\link{mlr_tuners_cmaes}}, \code{\link{mlr_tuners_design_points}}, \code{\link{mlr_tuners_gensa}}, +\code{\link{mlr_tuners_irace}}, \code{\link{mlr_tuners_nloptr}}, \code{\link{mlr_tuners_random_search}} } diff --git a/man/mlr_tuners_irace.Rd b/man/mlr_tuners_irace.Rd index cc4aff26a..f8ef6e116 100644 --- a/man/mlr_tuners_irace.Rd +++ b/man/mlr_tuners_irace.Rd @@ -5,42 +5,20 @@ \alias{TunerIrace} \title{TunerIrace} \description{ -Subclass for iterated racing tuning calling \code{\link[irace:irace]{irace::irace()}} from package \CRANpkg{irace}. +\code{TunerIrace} class that implements iterated racing. Calls \code{\link[irace:irace]{irace::irace()}} +from package \CRANpkg{irace}. } \section{Parameters}{ -\itemize{ -\item \code{show.irace.output} (\code{logical(1)}) -\item \code{debugLevel} (\code{integer(1)}) -\item \code{seed} (\code{integer(1)}) -\item \code{postselection} (\code{numeric(1)}) -\item \code{elitist} (\code{integer(1)}) -\item \code{elitistLimit} (\code{integer(1)}) -\item \code{nbIterations} (\code{integer(1)}) -\item \code{nbExperimentsPerIteration} (\code{integer(1)}) -\item \code{minNbSurvival} (\code{integer(1)}) -\item \code{nbConfigurations} (\code{integer(1)}) -\item \code{mu} (\code{integer(1)}) -\item \code{softRestart} (\code{integer(1)}) -\item \code{softRestartThreshold} (\code{numeric(1)}) -\item \code{digits} (\code{integer(1)}) -\item \code{testType} (\code{character(1)}) -\item \code{firstTest} (\code{integer(1)}) -\item \code{eachTest} (\code{integer(1)}) -\item \code{confidence} (\code{numeric(1)}) -\item \code{capping} (\code{integer(1)}) -\item \code{cappingType} (\code{character(1)}) -\item \code{boundType} (\code{character(1)}) -\item \code{boundMax} (\code{numeric(1)}) -\item \code{boundDigits} (\code{integer(1)}) -\item \code{boundPar} (\code{numeric(1)}) -\item \code{boundAsTimeout} (\code{numeric(1)}) +\describe{ +\item{\code{n_instances}}{\code{integer(1)}\cr +Number of resampling instances.} } -\code{show.irace.output} suppresses the output from \code{\link[irace:irace]{irace::irace()}} and only prints output from this package. For the meaning of all other parameters, see \code{\link[irace:defaultScenario]{irace::defaultScenario()}}. -Note that we have removed all control parameters which refer to the termination of the algorithm and -where our terminators allow to obtain the same behavior. +Note that we have removed all control parameters which refer to the +termination of the algorithm. Use \link{TerminatorRunTime} or \link{TerminatorEvals} +instead. Other terminators do not work with \code{TunerIrace}. } \section{Dictionary}{ @@ -53,7 +31,30 @@ tnr("irace") } \examples{ -# see ?Tuner +library(mlr3) +library(paradox) +search_space = ParamSet$new(list( + ParamDbl$new("cp", lower = 0.001, upper = 0.1) +)) +terminator = trm("evals", n_evals = 42) +instance = TuningInstanceSingleCrit$new( + task = tsk("iris"), + learner = lrn("classif.rpart"), + resampling = rsmp("holdout"), + measure = msr("classif.ce"), + search_space = search_space, + terminator = terminator +) +tt = tnr("irace") + +# modifies the instance by reference +tt$optimize(instance) + +# returns best configuration and best performance +instance$result + +# allows access of data.table of full path of all evaluations +instance$archive } \references{ Birattari, M., Stützle, T., Paquete, L., & Varrentrapp, K. (2002). diff --git a/man/mlr_tuners_nloptr.Rd b/man/mlr_tuners_nloptr.Rd index 06a04f010..8bedbcd69 100644 --- a/man/mlr_tuners_nloptr.Rd +++ b/man/mlr_tuners_nloptr.Rd @@ -24,7 +24,8 @@ their package defaults. To deactivate these conditions, set them to \code{-1}. \section{Dictionary}{ This \link{Tuner} can be instantiated via the \link[mlr3misc:Dictionary]{dictionary} -\link{mlr_tuners} or with the associated sugar function \code{\link[=tnr]{tnr()}}:\preformatted{mlr_tuners$get("nloptr") +\link{mlr_tuners} or with the associated sugar function \code{\link[=tnr]{tnr()}}:\preformatted{TunerNLoptr$new() +mlr_tuners$get("nloptr") tnr("nloptr") } } @@ -99,6 +100,7 @@ Other Tuner: \code{\link{mlr_tuners_design_points}}, \code{\link{mlr_tuners_gensa}}, \code{\link{mlr_tuners_grid_search}}, +\code{\link{mlr_tuners_irace}}, \code{\link{mlr_tuners_random_search}} } \concept{Tuner} diff --git a/man/mlr_tuners_random_search.Rd b/man/mlr_tuners_random_search.Rd index b9f44a9ad..68acfbae2 100644 --- a/man/mlr_tuners_random_search.Rd +++ b/man/mlr_tuners_random_search.Rd @@ -18,7 +18,8 @@ The random points are sampled by \code{\link[paradox:generate_design_random]{par \section{Dictionary}{ This \link{Tuner} can be instantiated via the \link[mlr3misc:Dictionary]{dictionary} -\link{mlr_tuners} or with the associated sugar function \code{\link[=tnr]{tnr()}}:\preformatted{mlr_tuners$get("random_search") +\link{mlr_tuners} or with the associated sugar function \code{\link[=tnr]{tnr()}}:\preformatted{TunerRandomSearch$new() +mlr_tuners$get("random_search") tnr("random_search") } } @@ -88,6 +89,7 @@ Other Tuner: \code{\link{mlr_tuners_design_points}}, \code{\link{mlr_tuners_gensa}}, \code{\link{mlr_tuners_grid_search}}, +\code{\link{mlr_tuners_irace}}, \code{\link{mlr_tuners_nloptr}} } \concept{Tuner} From 69324443285d7aaf2489cd8a040afe431fdc9fcd Mon Sep 17 00:00:00 2001 From: Marc Becker Date: Fri, 15 Jan 2021 12:02:45 +0100 Subject: [PATCH 28/40] Update title --- R/TunerIrace.R | 4 ++-- man/mlr_tuners_irace.Rd | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/R/TunerIrace.R b/R/TunerIrace.R index 6a5862afa..6d59f777c 100644 --- a/R/TunerIrace.R +++ b/R/TunerIrace.R @@ -1,7 +1,7 @@ -#' @title TunerIrace +#' @title Tuning via Iterated Racing. #' -#' @name mlr_tuners_irace #' @include Tuner.R +#' @name mlr_tuners_irace #' #' @description #' `TunerIrace` class that implements iterated racing. Calls [irace::irace()] diff --git a/man/mlr_tuners_irace.Rd b/man/mlr_tuners_irace.Rd index f8ef6e116..57cfc1fdf 100644 --- a/man/mlr_tuners_irace.Rd +++ b/man/mlr_tuners_irace.Rd @@ -3,7 +3,7 @@ \name{mlr_tuners_irace} \alias{mlr_tuners_irace} \alias{TunerIrace} -\title{TunerIrace} +\title{Tuning via Iterated Racing.} \description{ \code{TunerIrace} class that implements iterated racing. Calls \code{\link[irace:irace]{irace::irace()}} from package \CRANpkg{irace}. From 99a8511aab81f1428941ecee0e464c5992abc171 Mon Sep 17 00:00:00 2001 From: Marc Becker Date: Fri, 15 Jan 2021 12:23:03 +0100 Subject: [PATCH 29/40] Remove mlr3learners --- DESCRIPTION | 3 --- R/TunerIrace.R | 1 - tests/testthat.R | 3 +-- 3 files changed, 1 insertion(+), 6 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index d5dc630b2..6f54ebade 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -48,12 +48,9 @@ Imports: paradox (>= 0.6.0-9000), R6 Suggests: - e1071, adagio, GenSA, - glmnet, irace, - mlr3learners, mlr3pipelines, nloptr, rpart, diff --git a/R/TunerIrace.R b/R/TunerIrace.R index 6d59f777c..e90062cbb 100644 --- a/R/TunerIrace.R +++ b/R/TunerIrace.R @@ -53,7 +53,6 @@ #' #' # allows access of data.table of full path of all evaluations #' instance$archive - TunerIrace = R6Class("TunerIrace", inherit = Tuner, public = list( diff --git a/tests/testthat.R b/tests/testthat.R index 930a8b84b..7fdf65d97 100644 --- a/tests/testthat.R +++ b/tests/testthat.R @@ -3,7 +3,6 @@ if (requireNamespace("testthat", quietly = TRUE)) { library(checkmate) library(mlr3) library(mlr3tuning) - library(mlr3learners) - + test_check("mlr3tuning") } From 1929b38c0b28a940045f62c337645fc802a64eff Mon Sep 17 00:00:00 2001 From: Marc Becker Date: Fri, 15 Jan 2021 13:39:15 +0100 Subject: [PATCH 30/40] Change source --- R/TunerIrace.R | 6 ++---- R/bibentries.R | 11 ++++++++++- man/mlr_tuners_irace.Rd | 11 ++++++----- 3 files changed, 18 insertions(+), 10 deletions(-) diff --git a/R/TunerIrace.R b/R/TunerIrace.R index e90062cbb..cc1922a06 100644 --- a/R/TunerIrace.R +++ b/R/TunerIrace.R @@ -21,10 +21,8 @@ #' @templateVar id irace #' @template section_dictionary_tuners #' -#' @references -#' Birattari, M., Stützle, T., Paquete, L., & Varrentrapp, K. (2002). -#' A Racing Algorithm for Configuring Metaheuristics. -#' In Gecco (Vol. 2). +#' @source +#' `r format_bib("lopez_2016")` #' #' @family Tuner #' @export diff --git a/R/bibentries.R b/R/bibentries.R index 1e04a98ac..3849a9bf6 100644 --- a/R/bibentries.R +++ b/R/bibentries.R @@ -12,7 +12,6 @@ bibentries = c( doi = "10.32614/rj-2013-002" ), - tsallis_1996 = bibentry("article", title = "Generalized simulated annealing", author = "Constantino Tsallis and Daniel A. Stariolo", @@ -42,5 +41,15 @@ bibentries = c( author = "Johnson, Steven G", url = "https://github.com/stevengj/nlopt", year = "2020" + ), + + lopez_2016 = bibentry("article", + title = "The irace package: Iterated racing for automatic algorithm configuration", + author = "Manuel López-Ibáñez and Jérémie Dubois-Lacoste and Leslie Pérez Cáceres and Mauro Birattari and Thomas Stützle", + year = "2016", + journal = "Operations Research Perspectives", + volume = "3", + pages = "43--58", + doi = "https://doi.org/10.1016/j.orp.2016.09.002" ) ) diff --git a/man/mlr_tuners_irace.Rd b/man/mlr_tuners_irace.Rd index 57cfc1fdf..f7c23b3ff 100644 --- a/man/mlr_tuners_irace.Rd +++ b/man/mlr_tuners_irace.Rd @@ -4,6 +4,12 @@ \alias{mlr_tuners_irace} \alias{TunerIrace} \title{Tuning via Iterated Racing.} +\source{ +López-Ibáñez M, Dubois-Lacoste J, Cáceres LP, Birattari M, Stützle T (2016). +\dQuote{The irace package: Iterated racing for automatic algorithm configuration.} +\emph{Operations Research Perspectives}, \bold{3}, 43--58. +\doi{https://doi.org/10.1016/j.orp.2016.09.002}. +} \description{ \code{TunerIrace} class that implements iterated racing. Calls \code{\link[irace:irace]{irace::irace()}} from package \CRANpkg{irace}. @@ -56,11 +62,6 @@ instance$result # allows access of data.table of full path of all evaluations instance$archive } -\references{ -Birattari, M., Stützle, T., Paquete, L., & Varrentrapp, K. (2002). -A Racing Algorithm for Configuring Metaheuristics. -In Gecco (Vol. 2). -} \seealso{ Other Tuner: \code{\link{mlr_tuners_cmaes}}, From 908d90fe4d4c377c3efa38cf3ebe32a14800029d Mon Sep 17 00:00:00 2001 From: Marc Becker Date: Fri, 15 Jan 2021 14:04:24 +0100 Subject: [PATCH 31/40] Use try in unit test --- tests/testthat/test_TunerIrace.R | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/testthat/test_TunerIrace.R b/tests/testthat/test_TunerIrace.R index a8fd87536..8112a97d5 100644 --- a/tests/testthat/test_TunerIrace.R +++ b/tests/testthat/test_TunerIrace.R @@ -17,7 +17,10 @@ test_that("TunerIrace works with TerminatorRunTime", { instance = TuningInstanceSingleCrit$new(tsk("iris"), lrn("classif.rpart"), rsmp("holdout"), msr("classif.ce"), trm("run_time", secs = 30), search_space) tuner = tnr("irace") - expect_data_table(tuner$optimize(instance)) + # TunerIrace sometimes returns no result if used with TerminatorRunTime. + # Therefore we can only check if the archive contains evaluations. + try(tuner$optimize(instance), silent = TRUE) + expect_true(nrow(instance$archive$data) > 0) }) test_that("TunerIrace works with dependencies", { From 8079a63222aca41cdd2f95b055f8931ec08869ce Mon Sep 17 00:00:00 2001 From: Marc Becker Date: Fri, 15 Jan 2021 14:04:38 +0100 Subject: [PATCH 32/40] Remove non-ASCII characters --- R/bibentries.R | 2 +- man/mlr_tuners_irace.Rd | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/R/bibentries.R b/R/bibentries.R index 3849a9bf6..d735727f0 100644 --- a/R/bibentries.R +++ b/R/bibentries.R @@ -45,7 +45,7 @@ bibentries = c( lopez_2016 = bibentry("article", title = "The irace package: Iterated racing for automatic algorithm configuration", - author = "Manuel López-Ibáñez and Jérémie Dubois-Lacoste and Leslie Pérez Cáceres and Mauro Birattari and Thomas Stützle", + author = "Manuel Lopez-Ibanez and Jeremie Dubois-Lacoste and Leslie Perez Caceres and Mauro Birattari and Thomas Stuetzle", year = "2016", journal = "Operations Research Perspectives", volume = "3", diff --git a/man/mlr_tuners_irace.Rd b/man/mlr_tuners_irace.Rd index f7c23b3ff..bb9d5fdac 100644 --- a/man/mlr_tuners_irace.Rd +++ b/man/mlr_tuners_irace.Rd @@ -5,7 +5,7 @@ \alias{TunerIrace} \title{Tuning via Iterated Racing.} \source{ -López-Ibáñez M, Dubois-Lacoste J, Cáceres LP, Birattari M, Stützle T (2016). +Lopez-Ibanez M, Dubois-Lacoste J, Caceres LP, Birattari M, Stuetzle T (2016). \dQuote{The irace package: Iterated racing for automatic algorithm configuration.} \emph{Operations Research Perspectives}, \bold{3}, 43--58. \doi{https://doi.org/10.1016/j.orp.2016.09.002}. From 33ba13323517b215c4c3a06358421a8345c2e5a2 Mon Sep 17 00:00:00 2001 From: Marc Becker Date: Mon, 18 Jan 2021 16:06:10 +0100 Subject: [PATCH 33/40] Review fixes --- R/ObjectiveTuning.R | 2 +- R/TunerIrace.R | 14 +++++++++----- R/irace_helpers.R | 5 ++--- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/R/ObjectiveTuning.R b/R/ObjectiveTuning.R index be100c88d..b8b4c6c5d 100644 --- a/R/ObjectiveTuning.R +++ b/R/ObjectiveTuning.R @@ -44,7 +44,7 @@ ObjectiveTuning = R6Class("ObjectiveTuning", self$task = assert_task(as_task(task, clone = TRUE)) self$learner = assert_learner(as_learner(learner, clone = TRUE)) - self$resampling = assert_resampling(as_resampling(resampling, clone = TRUE)) + self$resampling = resampling self$measures = assert_measures(as_measures(measures, clone = TRUE), task = self$task, learner = self$learner) self$store_benchmark_result = assert_logical(store_benchmark_result) diff --git a/R/TunerIrace.R b/R/TunerIrace.R index cc1922a06..2013d3a41 100644 --- a/R/TunerIrace.R +++ b/R/TunerIrace.R @@ -101,6 +101,11 @@ TunerIrace = R6Class("TunerIrace", pv = self$param_set$values terminator = inst$terminator objective = inst$objective + + # Check terminators + if (!(inherits(terminator, "TerminatorEvals") || inherits(terminator, "TerminatorRunTime"))) { + stopf("%s is not supported. Use or instead.", format(inst$terminator)) + } # Set resampling instances ri = map(seq(pv$n_instances), function(n) { @@ -111,12 +116,12 @@ TunerIrace = R6Class("TunerIrace", # Make scenario scenario = c(list( - targetRunner = targetRunner, + targetRunner = target_runner, logFile = tempfile(), instances = ri, debugLevel = 0, - maxExperiments = if (class(inst$terminator)[1] == "TerminatorEvals") terminator$param_set$values$n_evals else 0, - maxTime = if (class(inst$terminator)[1] == "TerminatorRunTime") terminator$param_set$values$secs else 0, + maxExperiments = if (inherits(terminator, "TerminatorEvals")) terminator$param_set$values$n_evals else 0, + maxTime = if (inherits(terminator, "TerminatorRunTime")) terminator$param_set$values$secs - 2 else 0, targetRunnerData = list(inst = inst) ), pv) @@ -133,8 +138,7 @@ TunerIrace = R6Class("TunerIrace", if(length(private$.result_id) == 0) { stop("irace::irace did not return a result. The evaluated configurations are still accessible through the archive.") } - id_configuration = NULL - res = inst$archive$data[id_configuration == private$.result_id, ] + res = inst$archive$data[get("id_configuration") == private$.result_id, ] cols = c(inst$archive$cols_x, "id_configuration") xdt = res[1, cols, with = FALSE] y = set_names(mean(unlist(res[, inst$archive$cols_y, with = FALSE])), inst$archive$cols_y) diff --git a/R/irace_helpers.R b/R/irace_helpers.R index f4e689cee..d769fed1e 100644 --- a/R/irace_helpers.R +++ b/R/irace_helpers.R @@ -57,7 +57,7 @@ get_irace_condition = function(ps) { return(tab) } -targetRunner = function(experiment, scenario) { # nolint +target_runner = function(experiment, scenario) { # nolint t0 = Sys.time() tuning_instance = scenario$targetRunnerData$inst @@ -79,7 +79,6 @@ targetRunner = function(experiment, scenario) { # nolint # evaluate configuration # objective_function cannot pass extra information cost = as.numeric(tuning_instance$eval_batch(cbind(config, extra))) * tuning_instance$objective_multiplicator - t1 = Sys.time() - return(list(cost = cost, time = as.numeric(t1 - t0))) + return(list(cost = cost, time = as.numeric(difftime(Sys.time(), t0, units = "secs")))) } From 54f06b4d0a64171971810e189528bffd4b93673f Mon Sep 17 00:00:00 2001 From: Marc Becker Date: Mon, 18 Jan 2021 16:18:01 +0100 Subject: [PATCH 34/40] Remove 5 seconds and update documentation --- R/TunerIrace.R | 11 ++++++----- man-roxygen/param_resampling.R | 4 +++- man/ObjectiveTuning.Rd | 4 +++- man/TuningInstanceMultiCrit.Rd | 4 +++- man/TuningInstanceSingleCrit.Rd | 4 +++- man/mlr_tuners_irace.Rd | 9 +++++---- tests/testthat/test_TunerIrace.R | 2 +- 7 files changed, 24 insertions(+), 14 deletions(-) diff --git a/R/TunerIrace.R b/R/TunerIrace.R index 2013d3a41..56192e5ba 100644 --- a/R/TunerIrace.R +++ b/R/TunerIrace.R @@ -13,10 +13,11 @@ #' Number of resampling instances.} #' } #' -#' For the meaning of all other parameters, see [irace::defaultScenario()]. -#' Note that we have removed all control parameters which refer to the -#' termination of the algorithm. Use [TerminatorRunTime] or [TerminatorEvals] -#' instead. Other terminators do not work with `TunerIrace`. +#' For the meaning of all other parameters, see [irace::defaultScenario()]. Note +#' that we have removed all control parameters which refer to the termination of +#' the algorithm. Use [TerminatorRunTime] or [TerminatorEvals] instead. Other +#' terminators do not work with `TunerIrace`. We substract 5 seconds from the +#' [TerminatorRunTime] budget for stability reasons. #' #' @templateVar id irace #' @template section_dictionary_tuners @@ -121,7 +122,7 @@ TunerIrace = R6Class("TunerIrace", instances = ri, debugLevel = 0, maxExperiments = if (inherits(terminator, "TerminatorEvals")) terminator$param_set$values$n_evals else 0, - maxTime = if (inherits(terminator, "TerminatorRunTime")) terminator$param_set$values$secs - 2 else 0, + maxTime = if (inherits(terminator, "TerminatorRunTime")) terminator$param_set$values$secs - 5 else 0, targetRunnerData = list(inst = inst) ), pv) diff --git a/man-roxygen/param_resampling.R b/man-roxygen/param_resampling.R index 8f6bb2d66..45ff73b84 100644 --- a/man-roxygen/param_resampling.R +++ b/man-roxygen/param_resampling.R @@ -1,3 +1,5 @@ #' @param resampling ([mlr3::Resampling])\cr #' Uninstantiated resamplings are instantiated during construction -#' so that all configurations are evaluated on the same data splits. +#' so that all configurations are evaluated on the same data splits. If a new +#' resampling is passed, it is instantiated with new data splits. Already +#' instantiated resamplings are kept unchanged. diff --git a/man/ObjectiveTuning.Rd b/man/ObjectiveTuning.Rd index 0b0cc5ce5..aa875b2e6 100644 --- a/man/ObjectiveTuning.Rd +++ b/man/ObjectiveTuning.Rd @@ -80,7 +80,9 @@ Task to operate on.} \item{\code{resampling}}{(\link[mlr3:Resampling]{mlr3::Resampling})\cr Uninstantiated resamplings are instantiated during construction -so that all configurations are evaluated on the same data splits.} +so that all configurations are evaluated on the same data splits. If a new +resampling is passed, it is instantiated with new data splits. Already +instantiated resamplings are kept unchanged.} \item{\code{measures}}{(list of \link[mlr3:Measure]{mlr3::Measure})\cr Measures to optimize. diff --git a/man/TuningInstanceMultiCrit.Rd b/man/TuningInstanceMultiCrit.Rd index afe5ed613..12e332c78 100644 --- a/man/TuningInstanceMultiCrit.Rd +++ b/man/TuningInstanceMultiCrit.Rd @@ -83,7 +83,9 @@ Task to operate on.} \item{\code{resampling}}{(\link[mlr3:Resampling]{mlr3::Resampling})\cr Uninstantiated resamplings are instantiated during construction -so that all configurations are evaluated on the same data splits.} +so that all configurations are evaluated on the same data splits. If a new +resampling is passed, it is instantiated with new data splits. Already +instantiated resamplings are kept unchanged.} \item{\code{measures}}{(list of \link[mlr3:Measure]{mlr3::Measure})\cr Measures to optimize. diff --git a/man/TuningInstanceSingleCrit.Rd b/man/TuningInstanceSingleCrit.Rd index 3e79657e4..f195970b0 100644 --- a/man/TuningInstanceSingleCrit.Rd +++ b/man/TuningInstanceSingleCrit.Rd @@ -158,7 +158,9 @@ Task to operate on.} \item{\code{resampling}}{(\link[mlr3:Resampling]{mlr3::Resampling})\cr Uninstantiated resamplings are instantiated during construction -so that all configurations are evaluated on the same data splits.} +so that all configurations are evaluated on the same data splits. If a new +resampling is passed, it is instantiated with new data splits. Already +instantiated resamplings are kept unchanged.} \item{\code{measure}}{(\link[mlr3:Measure]{mlr3::Measure})\cr Measure to optimize.} diff --git a/man/mlr_tuners_irace.Rd b/man/mlr_tuners_irace.Rd index bb9d5fdac..5d4c6f14f 100644 --- a/man/mlr_tuners_irace.Rd +++ b/man/mlr_tuners_irace.Rd @@ -21,10 +21,11 @@ from package \CRANpkg{irace}. Number of resampling instances.} } -For the meaning of all other parameters, see \code{\link[irace:defaultScenario]{irace::defaultScenario()}}. -Note that we have removed all control parameters which refer to the -termination of the algorithm. Use \link{TerminatorRunTime} or \link{TerminatorEvals} -instead. Other terminators do not work with \code{TunerIrace}. +For the meaning of all other parameters, see \code{\link[irace:defaultScenario]{irace::defaultScenario()}}. Note +that we have removed all control parameters which refer to the termination of +the algorithm. Use \link{TerminatorRunTime} or \link{TerminatorEvals} instead. Other +terminators do not work with \code{TunerIrace}. We substract 5 seconds from the +\link{TerminatorRunTime} budget for stability reasons. } \section{Dictionary}{ diff --git a/tests/testthat/test_TunerIrace.R b/tests/testthat/test_TunerIrace.R index 8112a97d5..d0b641c39 100644 --- a/tests/testthat/test_TunerIrace.R +++ b/tests/testthat/test_TunerIrace.R @@ -19,7 +19,7 @@ test_that("TunerIrace works with TerminatorRunTime", { tuner = tnr("irace") # TunerIrace sometimes returns no result if used with TerminatorRunTime. # Therefore we can only check if the archive contains evaluations. - try(tuner$optimize(instance), silent = TRUE) + tuner$optimize(instance) expect_true(nrow(instance$archive$data) > 0) }) From 32c53dbd0ee59a0c2d10cdff8704b2e839a68040 Mon Sep 17 00:00:00 2001 From: Marc Becker Date: Mon, 18 Jan 2021 16:40:11 +0100 Subject: [PATCH 35/40] Add unit test --- tests/testthat/test_TunerIrace.R | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/tests/testthat/test_TunerIrace.R b/tests/testthat/test_TunerIrace.R index d0b641c39..555064fe1 100644 --- a/tests/testthat/test_TunerIrace.R +++ b/tests/testthat/test_TunerIrace.R @@ -17,10 +17,17 @@ test_that("TunerIrace works with TerminatorRunTime", { instance = TuningInstanceSingleCrit$new(tsk("iris"), lrn("classif.rpart"), rsmp("holdout"), msr("classif.ce"), trm("run_time", secs = 30), search_space) tuner = tnr("irace") - # TunerIrace sometimes returns no result if used with TerminatorRunTime. - # Therefore we can only check if the archive contains evaluations. - tuner$optimize(instance) - expect_true(nrow(instance$archive$data) > 0) + expect_data_table(tuner$optimize(instance)) +}) + +test_that("TunerIrace with unsupported terminators", { + search_space = ps( + cp = p_dbl(lower = 0.001, upper = 0.1), + minsplit = p_int(lower = 1, upper = 10) + ) + instance = TuningInstanceSingleCrit$new(tsk("iris"), lrn("classif.rpart"), rsmp("holdout"), msr("classif.ce"), + trm("run_time", secs = 30), search_space) + tuner = tnr("irace") }) test_that("TunerIrace works with dependencies", { From 5248a9b60890d297aa78a450f0a8ec63b8405893 Mon Sep 17 00:00:00 2001 From: Marc Becker Date: Mon, 18 Jan 2021 17:12:29 +0100 Subject: [PATCH 36/40] Unit test --- tests/testthat/test_TunerIrace.R | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/testthat/test_TunerIrace.R b/tests/testthat/test_TunerIrace.R index 555064fe1..4d8030450 100644 --- a/tests/testthat/test_TunerIrace.R +++ b/tests/testthat/test_TunerIrace.R @@ -21,13 +21,16 @@ test_that("TunerIrace works with TerminatorRunTime", { }) test_that("TunerIrace with unsupported terminators", { - search_space = ps( + search_space = ps( cp = p_dbl(lower = 0.001, upper = 0.1), minsplit = p_int(lower = 1, upper = 10) ) instance = TuningInstanceSingleCrit$new(tsk("iris"), lrn("classif.rpart"), rsmp("holdout"), msr("classif.ce"), - trm("run_time", secs = 30), search_space) + trm("perf_reached", level = 0.1), search_space) tuner = tnr("irace") + expect_error(tuner$optimize(instance), + regex = " is not supported. Use or instead.", + fixed = TRUE) }) test_that("TunerIrace works with dependencies", { From dcdac67785e64e752e40f8b3f868b3e3a0719c38 Mon Sep 17 00:00:00 2001 From: Marc Becker Date: Wed, 20 Jan 2021 15:53:27 +0100 Subject: [PATCH 37/40] Make mlr3 and paradox depends --- DESCRIPTION | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index 6f54ebade..b2facf2fe 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -37,15 +37,15 @@ URL: https://mlr3tuning.mlr-org.com, https://github.com/mlr-org/mlr3tuning BugReports: https://github.com/mlr-org/mlr3tuning/issues Depends: + mlr3 (>= 0.7.0), + paradox (>= 0.6.0-9000), R (>= 3.1.0) Imports: bbotk (>= 0.2.2.9000), checkmate (>= 2.0.0), data.table, lgr, - mlr3 (>= 0.7.0), mlr3misc (>= 0.7.0), - paradox (>= 0.6.0-9000), R6 Suggests: adagio, @@ -58,10 +58,10 @@ Suggests: Remotes: mlr-org/bbotk, mlr-org/paradox -Encoding: UTF-8 -NeedsCompilation: no Config/testthat/edition: 3 Config/testthat/parallel: true +Encoding: UTF-8 +NeedsCompilation: no Roxygen: list(markdown = TRUE) RoxygenNote: 7.1.1 Collate: From 315d57415220fa32ab6ccbeb221321ab0f730b05 Mon Sep 17 00:00:00 2001 From: Marc Becker Date: Tue, 2 Feb 2021 21:49:09 +0100 Subject: [PATCH 38/40] Use replicate --- R/TunerIrace.R | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/R/TunerIrace.R b/R/TunerIrace.R index 56192e5ba..0297d6c77 100644 --- a/R/TunerIrace.R +++ b/R/TunerIrace.R @@ -108,8 +108,9 @@ TunerIrace = R6Class("TunerIrace", stopf("%s is not supported. Use or instead.", format(inst$terminator)) } + browser() # Set resampling instances - ri = map(seq(pv$n_instances), function(n) { + ri = replicate(seq(pv$n_instances), { r = objective$resampling$clone() r$instantiate(objective$task) }) From 655b6f8fc576e5caa6a7370931a80e383d3d2dbb Mon Sep 17 00:00:00 2001 From: Marc Becker <33069354+be-marc@users.noreply.github.com> Date: Wed, 3 Feb 2021 11:04:18 +0100 Subject: [PATCH 39/40] Update R/TunerIrace.R Co-authored-by: Jakob Richter --- R/TunerIrace.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/TunerIrace.R b/R/TunerIrace.R index 0297d6c77..c6f9f8ab5 100644 --- a/R/TunerIrace.R +++ b/R/TunerIrace.R @@ -110,7 +110,7 @@ TunerIrace = R6Class("TunerIrace", browser() # Set resampling instances - ri = replicate(seq(pv$n_instances), { + ri = replicate(pv$n_instances, { r = objective$resampling$clone() r$instantiate(objective$task) }) From 74ea4435cad587968bf6f7c1b91db336265d9b58 Mon Sep 17 00:00:00 2001 From: Marc Becker Date: Thu, 4 Feb 2021 10:15:27 +0100 Subject: [PATCH 40/40] Remove browser --- R/TunerIrace.R | 1 - 1 file changed, 1 deletion(-) diff --git a/R/TunerIrace.R b/R/TunerIrace.R index c6f9f8ab5..e666a81fd 100644 --- a/R/TunerIrace.R +++ b/R/TunerIrace.R @@ -108,7 +108,6 @@ TunerIrace = R6Class("TunerIrace", stopf("%s is not supported. Use or instead.", format(inst$terminator)) } - browser() # Set resampling instances ri = replicate(pv$n_instances, { r = objective$resampling$clone()