diff --git a/NEWS.md b/NEWS.md index eb7ab7a7..8c85d1d3 100644 --- a/NEWS.md +++ b/NEWS.md @@ -57,6 +57,11 @@ `repairConfiguration` function (see `defaultScenario()`) only needs two arguments: `configuration` and `parameters`. See examples in the user-guide. + * New options `trainInstancesText`, `testInstancesText`, `parameterText`, and + `configurationsText` which accepts the content that is normally in + `trainInstancesFile`, `testInstancesFile`, `parameterText`, and + `configurationsText` as a string. (Deyao Chen) + ## Fixes * Fix (#44): irace now will give an error if the domain of real-valued (r) diff --git a/R/irace-options.R b/R/irace-options.R index 1b89605c..f5d2262c 100644 --- a/R/irace-options.R +++ b/R/irace-options.R @@ -5,13 +5,14 @@ # FIXME: Add special type for R functions. # FIXME: For i and r add their domain. .irace.params.def <- structure(list(name = c(".help", ".version", ".check", ".init", -".onlytest", "scenarioFile", "execDir", "parameterFile", "initConfigurations", -"configurationsFile", "logFile", "recoveryFile", "instances", -"trainInstancesDir", "trainInstancesFile", "sampleInstances", -"testInstancesDir", "testInstancesFile", "testInstances", "testNbElites", -"testIterationElites", "testType", "firstTest", "blockSize", -"eachTest", "targetRunner", "targetRunnerLauncher", "targetCmdline", -"targetRunnerRetries", "targetRunnerData", "targetRunnerParallel", +".onlytest", "scenarioFile", "execDir", "parameterFile", "parameterText", +"initConfigurations", "configurationsFile", "configurationsText", +"logFile", "recoveryFile", "instances", "trainInstancesDir", +"trainInstancesFile", "trainInstancesText", "sampleInstances", +"testInstancesDir", "testInstancesFile", "testInstancesText", +"testInstances", "testNbElites", "testIterationElites", "testType", +"firstTest", "blockSize", "eachTest", "targetRunner", "targetRunnerLauncher", +"targetCmdline", "targetRunnerRetries", "targetRunnerData", "targetRunnerParallel", "targetEvaluator", "deterministic", "maxExperiments", "maxTime", "budgetEstimation", "minMeasurableTime", "parallel", "loadBalancing", "mpi", "batchmode", "digits", "quiet", "debugLevel", "seed", @@ -20,25 +21,27 @@ "boundType", "boundMax", "boundDigits", "boundPar", "boundAsTimeout", "postselection", "aclib", "nbIterations", "nbExperimentsPerIteration", "minNbSurvival", "nbConfigurations", "mu", "confidence"), type = c("x", -"x", "x", "x", "p", "p", "p", "p", "x", "p", "p", "p", "s", "p", -"p", "b", "p", "p", "x", "i", "b", "s", "i", "i", "i", "p", "p", -"s", "i", "x", "x", "p", "b", "i", "i", "r", "r", "i", "b", "b", -"s", "i", "b", "i", "i", "b", "r", "b", "i", "i", "x", "b", "s", -"s", "i", "i", "i", "b", "r", "b", "i", "i", "i", "i", "i", "r" -), short = c("-h", "-v", "-c", "-i", "", "-s", "", "-p", "", -"", "-l", "", "", "", "", "", "", "", "", "", "", "", "", "", +"x", "x", "x", "p", "p", "p", "p", "p", "x", "p", "p", "p", "p", +"s", "p", "p", "p", "b", "p", "p", "p", "x", "i", "b", "s", "i", +"i", "i", "p", "p", "s", "i", "x", "x", "p", "b", "i", "i", "r", +"r", "i", "b", "b", "s", "i", "b", "i", "i", "b", "r", "b", "i", +"i", "x", "b", "s", "s", "i", "i", "i", "b", "r", "b", "i", "i", +"i", "i", "i", "r"), short = c("-h", "-v", "-c", "-i", "", "-s", +"", "-p", "", "", "", "", "-l", "", "", "", "", "", "", "", "", +"", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", +"", "", "", "", "", "", "", "", "", "-q", "", "", "", "", "-e", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", -"", "", "-q", "", "", "", "", "-e", "", "", "", "", "", "", "", -"", "", "", "", "", "", "", "", "", "", ""), long = c("--help", -"--version", "--check", "--init", "--only-test", "--scenario", -"--exec-dir", "--parameter-file", "", "--configurations-file", +"", ""), long = c("--help", "--version", "--check", "--init", +"--only-test", "--scenario", "--exec-dir", "--parameter-file", +"--parameter-text", "", "--configurations-file", "--configurations-text", "--log-file", "--recovery-file", "", "--train-instances-dir", -"--train-instances-file", "--sample-instances", "--test-instances-dir", -"--test-instances-file", "", "--test-num-elites", "--test-iteration-elites", -"--test-type", "--first-test", "--block-size", "--each-test", -"--target-runner", "--target-runner-launcher", "--target-cmdline", -"--target-runner-retries", "", "", "--target-evaluator", "--deterministic", -"--max-experiments", "--max-time", "--budget-estimation", "--min-measurable-time", +"--train-instances-file", "--train-instances-text", "--sample-instances", +"--test-instances-dir", "--test-instances-file", "--test-instances-text", +"", "--test-num-elites", "--test-iteration-elites", "--test-type", +"--first-test", "--block-size", "--each-test", "--target-runner", +"--target-runner-launcher", "--target-cmdline", "--target-runner-retries", +"", "", "--target-evaluator", "--deterministic", "--max-experiments", +"--max-time", "--budget-estimation", "--min-measurable-time", "--parallel", "--load-balancing", "--mpi", "--batchmode", "--digits", "--quiet", "--debug-level", "--seed", "--soft-restart", "--soft-restart-threshold", "--elitist", "--elitist-new-instances", "--elitist-limit", "", @@ -47,29 +50,35 @@ "--aclib", "--iterations", "--experiments-per-iteration", "--min-survival", "--num-configurations", "--mu", "--confidence"), default = c(NA, NA, NA, "", "", "./scenario.txt", "./", "./parameters.txt", "", -"", "./irace.Rdata", "", "", "./Instances", "", "1", "", "", -"", "1", "0", "", "5", "1", "1", "./target-runner", "", "{configurationID} {instanceID} {seed} {instance} {bound} {targetRunnerArgs}", +"", "", "", "./irace.Rdata", "", "", "./Instances", "", "", "1", +"", "", "", "", "1", "0", "", "5", "1", "1", "./target-runner", +"", "{configurationID} {instanceID} {seed} {instance} {bound} {targetRunnerArgs}", "0", "", "", "", "0", "0", "0", "0.02", "0.01", "0", "1", "0", "0", "4", "0", "0", NA, "1", "", "1", "1", "2", "", NA, "median", "candidate", "0", "0", "1", "1", "0", "0", "0", "0", "0", "0", "5", "0.95"), domain = c(NA, NA, NA, NA, NA, NA, NA, NA, NA, -NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, "F-test,t-test,t-test-holm,t-test-bonferroni", NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, -NA, NA, "sge,pbs,torque,slurm,htcondor", NA, NA, NA, NA, NA, -NA, NA, NA, NA, NA, NA, "median,mean,worst,best", "instance,candidate", -NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA), description = c("Show this help.", -"Show irace package version.", "Check scenario.", "Initialize the working directory with template config files.", +"F-test,t-test,t-test-holm,t-test-bonferroni", NA, NA, NA, NA, +NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, "sge,pbs,torque,slurm,htcondor", +NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, "median,mean,worst,best", +"instance,candidate", NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, +NA, NA), description = c("Show this help.", "Show irace package version.", +"Check scenario.", "Initialize the working directory with template config files.", "Only test the configurations given in the file passed as argument.", "File that describes the configuration scenario setup and other irace settings.", "Directory where the programs will be run.", "File that contains the description of the parameters of the target algorithm.", +"A string that contains the description of the parameters of the target algorithm.", "", "File that contains a table of initial configurations. If empty or `NULL`, all initial configurations are randomly generated.", +"A string that contains a table of initial configurations. If empty or `NULL`, all initial configurations are randomly generated.", "File to save tuning results as an R dataset, either absolute path or relative to execDir.", "Previously saved log file to recover the execution of `irace`, either absolute path or relative to the current directory. If empty or `NULL`, recovery is not performed.", -"", "Directory where training instances are located; either absolute path or relative to current directory. If no `trainInstancesFiles` is provided, all the files in `trainInstancesDir` will be listed as instances.", +"", "Directory where training instances are located; either absolute path or relative to current directory. If no `trainInstancesFiles` or `trainInstancesText` is provided, all the files in `trainInstancesDir` will be listed as instances.", "File that contains a list of training instances and optionally additional parameters for them. If `trainInstancesDir` is provided, `irace` will search for the files in this folder.", +"A string that contains a list of training instances and optionally additional parameters for them. If `trainInstancesDir` is provided, `irace` will search for the files in this folder.", "Randomly sample the training instances or use them in the order given.", "Directory where testing instances are located, either absolute or relative to current directory.", "File containing a list of test instances and optionally additional parameters for them.", +"A string containing a list of test instances and optionally additional parameters for them.", "", "Number of elite configurations returned by irace that will be tested if test instances are provided.", "Enable/disable testing the elite configurations found at each iteration.", "Statistical test used for elimination. The default value selects `t-test` if `capping` is enabled or `F-test`, otherwise. Valid values are: F-test (Friedman test), t-test (pairwise t-tests with no correction), t-test-bonferroni (t-test with Bonferroni's correction for multiple comparisons), t-test-holm (t-test with Holm's correction for multiple comparisons).", @@ -114,28 +123,29 @@ NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA), description = c("Show this help "Parameter used to define the number of configurations sampled and evaluated at each iteration.", "Confidence level for the elimination test.")), row.names = c(".help", ".version", ".check", ".init", ".onlytest", "scenarioFile", "execDir", -"parameterFile", "initConfigurations", "configurationsFile", +"parameterFile", "parameterText", "initConfigurations", "configurationsFile", +"configurationsText", "logFile", "recoveryFile", "instances", +"trainInstancesDir", "trainInstancesFile", "trainInstancesText", +"sampleInstances", "testInstancesDir", "testInstancesFile", "testInstancesText", +"testInstances", "testNbElites", "testIterationElites", "testType", +"firstTest", "blockSize", "eachTest", "targetRunner", "targetRunnerLauncher", +"targetCmdline", "targetRunnerRetries", "targetRunnerData", "targetRunnerParallel", +"targetEvaluator", "deterministic", "maxExperiments", "maxTime", +"budgetEstimation", "minMeasurableTime", "parallel", "loadBalancing", +"mpi", "batchmode", "digits", "quiet", "debugLevel", "seed", +"softRestart", "softRestartThreshold", "elitist", "elitistNewInstances", +"elitistLimit", "repairConfiguration", "capping", "cappingType", +"boundType", "boundMax", "boundDigits", "boundPar", "boundAsTimeout", +"postselection", "aclib", "nbIterations", "nbExperimentsPerIteration", +"minNbSurvival", "nbConfigurations", "mu", "confidence"), class = "data.frame") +.irace.params.names <- c("scenarioFile", "execDir", "parameterFile", "parameterText", +"initConfigurations", "configurationsFile", "configurationsText", "logFile", "recoveryFile", "instances", "trainInstancesDir", -"trainInstancesFile", "sampleInstances", "testInstancesDir", -"testInstancesFile", "testInstances", "testNbElites", "testIterationElites", -"testType", "firstTest", "blockSize", "eachTest", "targetRunner", -"targetRunnerLauncher", "targetCmdline", "targetRunnerRetries", -"targetRunnerData", "targetRunnerParallel", "targetEvaluator", -"deterministic", "maxExperiments", "maxTime", "budgetEstimation", -"minMeasurableTime", "parallel", "loadBalancing", "mpi", "batchmode", -"digits", "quiet", "debugLevel", "seed", "softRestart", "softRestartThreshold", -"elitist", "elitistNewInstances", "elitistLimit", "repairConfiguration", -"capping", "cappingType", "boundType", "boundMax", "boundDigits", -"boundPar", "boundAsTimeout", "postselection", "aclib", "nbIterations", -"nbExperimentsPerIteration", "minNbSurvival", "nbConfigurations", -"mu", "confidence"), class = "data.frame") -.irace.params.names <- c("scenarioFile", "execDir", "parameterFile", "initConfigurations", -"configurationsFile", "logFile", "recoveryFile", "instances", -"trainInstancesDir", "trainInstancesFile", "sampleInstances", -"testInstancesDir", "testInstancesFile", "testInstances", "testNbElites", -"testIterationElites", "testType", "firstTest", "blockSize", -"eachTest", "targetRunner", "targetRunnerLauncher", "targetCmdline", -"targetRunnerRetries", "targetRunnerData", "targetRunnerParallel", +"trainInstancesFile", "trainInstancesText", "sampleInstances", +"testInstancesDir", "testInstancesFile", "testInstancesText", +"testInstances", "testNbElites", "testIterationElites", "testType", +"firstTest", "blockSize", "eachTest", "targetRunner", "targetRunnerLauncher", +"targetCmdline", "targetRunnerRetries", "targetRunnerData", "targetRunnerParallel", "targetEvaluator", "deterministic", "maxExperiments", "maxTime", "budgetEstimation", "minMeasurableTime", "parallel", "loadBalancing", "mpi", "batchmode", "digits", "quiet", "debugLevel", "seed", diff --git a/R/irace.R b/R/irace.R index 953f5374..33f955d8 100644 --- a/R/irace.R +++ b/R/irace.R @@ -479,13 +479,15 @@ allConfigurationsInit <- function(scenario, parameters) confs_from_file <- NULL if (!is.null.or.empty(scenario$configurationsFile)) { confs_from_file <- readConfigurationsFile(scenario$configurationsFile, - parameters, scenario$debugLevel) + parameters, scenario$debugLevel, + scenario$configurationsText) } if (!is.null.or.empty(initConfigurations)) { - if (!is.null.or.empty(scenario$configurationsFile) && !identical(initConfigurations, confs_from_file)) + if ((!is.null.or.empty(scenario$configurationsFile) || !is.null.or.empty(scenario$configurationsText)) + && !identical(initConfigurations, confs_from_file)) irace.warning("'initConfigurations' provided in 'scenario',", " thus ignoring configurations from file '", - scenario$configurationsFile, "'.") + scenario$configurationsFile, "'and text '", scenario$configurationsText, "'.") cat("# Adding", nrow(initConfigurations), "initial configuration(s)\n") fix_configurations(initConfigurations, parameters, debugLevel = scenario$debugLevel) } else { @@ -571,7 +573,8 @@ irace_common <- function(scenario, parameters, simple, output.width = 9999L) # Read parameters definition parameters <- readParameters (file = scenario$parameterFile, digits = scenario$digits, - debugLevel = debugLevel) + debugLevel = debugLevel, + text = scenario$parameterText) } else { parameters <- checkParameters(parameters) } diff --git a/R/main.R b/R/main.R index 99e6157f..9219fc95 100644 --- a/R/main.R +++ b/R/main.R @@ -66,11 +66,12 @@ cat_irace_license <- function() #' output. #' #' @details This function checks the correctness of the scenario, reads the -#' parameter space from \code{scenario$parameterFile}, invokes [irace()], -#' prints its results in various formatted ways, (optionally) calls -#' [psRace()] and, finally, evaluates the best configurations on the test -#' instances (if provided). If you want a lower-level interface that just -#' runs irace, please see function [irace()]. +#' parameter space from \code{scenario$parameterFile} or +#' \code{scenario$parameterText}, invokes [irace()], prints its results in +#' various formatted ways, (optionally) calls [psRace()] and, finally, +#' evaluates the best configurations on the test instances (if provided). +#' If you want a lower-level interface that just runs irace, +#' please see function [irace()]. #' #' @templateVar return_invisible TRUE #' @template return_irace @@ -206,8 +207,10 @@ testing_fromfile <- function(filename, scenario) if (!scenario$quiet) printScenario(scenario) irace.note("Reading parameter file '", scenario$parameterFile, "'.\n") + irace.note("Parsing parameter text '", scenario$parameterText, "'.\n") parameters <- readParameters (file = scenario$parameterFile, - digits = scenario$digits) + digits = scenario$digits, + text = scenario$parameterText) configurations <- readConfigurationsFile (filename, parameters) configurations <- cbind(.ID. = 1:nrow(configurations), configurations, @@ -255,9 +258,10 @@ testing_common <- function(configurations, scenario, parameters, iraceResults) #' \code{FALSE} otherwise. #' #' @details If the `parameters` argument is missing, then the parameters -#' will be read from the file `parameterFile` given by `scenario`. If -#' `parameters` is provided, then `parameterFile` will not be read. This function will -#' try to execute the target-algorithm. +#' will be read from the file `parameterFile` or string `parameterText` +#' given by `scenario`. If `parameters` is provided, then `parameterFile` +#' or `parameterText` will not be read. This function will try to execute +#' the target-algorithm. #' #' @seealso #' \describe{ @@ -280,11 +284,13 @@ checkIraceScenario <- function(scenario, parameters) irace.note("Reading parameter file '", scenario$parameterFile, "'.\n") parameters <- readParameters (file = scenario$parameterFile, digits = scenario$digits, - debugLevel = 2) - } else if (!is.null.or.empty(scenario$parameterFile)) { + debugLevel = 2, + text = scenario$parameterText) + } else if (!is.null.or.empty(scenario$parameterFile) || !is.null.or.empty(scenario$parameterText)) { if (!scenario$quiet) cat("# checkIraceScenario(): 'parameters' provided by user. ", - "Parameter file '", scenario$parameterFile, "' will be ignored\n", sep = "") + "Parameter file '", scenario$parameterFile, "' and parameter text '", + scenario$parameterText, "' will be ignored\n", sep = "") } checkParameters(parameters) irace.note("Checking target runner.\n") diff --git a/R/readConfiguration.R b/R/readConfiguration.R index 4e0d1fe4..15073936 100644 --- a/R/readConfiguration.R +++ b/R/readConfiguration.R @@ -355,14 +355,16 @@ setup_test_instances <- function(scenario) { if (is.null.or.empty(scenario[["testInstances"]])) { if (!is.null.or.empty(scenario$testInstancesDir) || - !is.null.or.empty(scenario$testInstancesFile)) { + !is.null.or.empty(scenario$testInstancesFile) || + !is.null.or.empty(scenario$trainInstancesText)) { scenario$testInstancesDir <- path_rel2abs(scenario$testInstancesDir) if (!is.null.or.empty(scenario$testInstancesFile)) { scenario$testInstancesFile <- path_rel2abs(scenario$testInstancesFile) } scenario[["testInstances"]] <- readInstances(instancesDir = scenario$testInstancesDir, - instancesFile = scenario$testInstancesFile) + instancesFile = scenario$testInstancesFile, + instancesText = scenario$testInstancesText) } else { scenario[["testInstances"]] <- NULL } @@ -558,7 +560,8 @@ checkScenario <- function(scenario = defaultScenario()) scenario$instances <- readInstances(instancesDir = scenario$trainInstancesDir, - instancesFile = scenario$trainInstancesFile) + instancesFile = scenario$trainInstancesFile, + instancesText = scenario$trainInstancesText) } # Testing instances @@ -803,6 +806,7 @@ printScenario <- function(scenario) #' \item Target algorithm parameters: #' \describe{ #' \item{`parameterFile`}{File that contains the description of the parameters of the target algorithm. (Default: `"./parameters.txt"`)} +#' \item{`parameterText`}{A string that contains the description of the parameters of the target algorithm. (Default: `""`)} #' \item{`digits`}{Maximum number of decimal places that are significant for numerical (real) parameters. (Default: `4`)} #' } #' \item Target algorithm execution: @@ -824,12 +828,14 @@ printScenario <- function(scenario) #' \describe{ #' \item{`initConfigurations`}{Data frame describing initial configurations (usually read from a file using \code{readConfigurations}). (Default: `""`)} #' \item{`configurationsFile`}{File that contains a table of initial configurations. If empty or \code{NULL}, all initial configurations are randomly generated. (Default: `""`)} +#' \item{`configurationsText`}{A string that contains a table of initial configurations. If empty or \code{NULL}, all initial configurations are randomly generated. (Default: `""`)} #' } #' \item Training instances: #' \describe{ #' \item{`instances`}{Character vector of the instances to be used in the \code{targetRunner}. (Default: `""`)} -#' \item{`trainInstancesDir`}{Directory where training instances are located; either absolute path or relative to current directory. If no \code{trainInstancesFiles} is provided, all the files in \code{trainInstancesDir} will be listed as instances. (Default: `"./Instances"`)} +#' \item{`trainInstancesDir`}{Directory where training instances are located; either absolute path or relative to current directory. If no \code{trainInstancesFiles} or \code{trainInstancesText} is provided, all the files in \code{trainInstancesDir} will be listed as instances. (Default: `"./Instances"`)} #' \item{`trainInstancesFile`}{File that contains a list of training instances and optionally additional parameters for them. If \code{trainInstancesDir} is provided, \code{irace} will search for the files in this folder. (Default: `""`)} +#' \item{`trainInstancesText`}{A string that contains a list of training instances and optionally additional parameters for them. If \code{trainInstancesDir} is provided, \code{irace} will search for the files in this folder. (Default: `""`)} #' \item{`blockSize`}{Number of training instances, that make up a `block' in \code{trainInstancesFile}. Elimination of configurations will only be performed after evaluating a complete block and never in the middle of a block. Each block typically contains one instance from each instance class (type or family). (Default: `1`)} #' } #' \item Tuning budget: @@ -864,6 +870,7 @@ printScenario <- function(scenario) #' \describe{ #' \item{`testInstancesDir`}{Directory where testing instances are located, either absolute or relative to current directory. (Default: `""`)} #' \item{`testInstancesFile`}{File containing a list of test instances and optionally additional parameters for them. (Default: `""`)} +#' \item{`testInstancesText`}{A string containing a list of test instances and optionally additional parameters for them. (Default: `""`)} #' \item{`testInstances`}{Character vector of the instances to be used in the \code{targetRunner} when executing the testing. (Default: `""`)} #' \item{`testNbElites`}{Number of elite configurations returned by irace that will be tested if test instances are provided. (Default: `1`)} #' \item{`testIterationElites`}{Enable/disable testing the elite configurations found at each iteration. (Default: `0`)} @@ -901,15 +908,22 @@ defaultScenario <- function(scenario = list(), scenario } -readInstances <- function(instancesDir = NULL, instancesFile = NULL) +readInstances <- function(instancesDir = NULL, instancesFile = NULL, instancesText = NULL) { - if (is.null.or.empty(instancesDir) && is.null.or.empty(instancesFile)) - irace.error("Both instancesDir and instancesFile are empty: No instances provided") + if (is.null.or.empty(instancesDir) && is.null.or.empty(instancesFile) && is.null.or.empty(instancesText)) + irace.error("All instancesDir, instancesFile, and instanceText are empty: No instances provided") instances <- NULL - + + if (!is.null.or.empty(instancesText)) { + if (!is.null.or.empty(instancesFile)) { + irace.warning("Both instanceText and instanceFile are set. instanceText Takes precedent so ignoring instanceFile.") + } + instancesFile <- strcat("text=", deparse(substitute(text))) + } + if (!is.null.or.empty(instancesFile)) { - file.check (instancesFile, readable = TRUE, text = "instance file") + if (is.null.or.empty(instancesText)) file.check (instancesFile, readable = TRUE, text = "instance file") # We do not warn if the last line does not finish with a newline. instances <- readLines (instancesFile, warn = FALSE) instances <- sub("#.*$", "", instances) # Remove comments diff --git a/inst/templates/scenario.txt.tmpl b/inst/templates/scenario.txt.tmpl index 1686505a..077e874c 100644 --- a/inst/templates/scenario.txt.tmpl +++ b/inst/templates/scenario.txt.tmpl @@ -13,17 +13,26 @@ ## algorithm. # parameterFile = "./parameters.txt" +## A string that contains the description of the parameters of the target +## algorithm. +# parameterText = "" + ## File that contains a table of initial configurations. If empty or `NULL`, ## all initial configurations are randomly generated. # configurationsFile = "" +## A string that contains a table of initial configurations. If empty or +## `NULL`, all initial configurations are randomly generated. +# configurationsText = "" + ## File to save tuning results as an R dataset, either absolute path or ## relative to execDir. # logFile = "./irace.Rdata" ## Directory where training instances are located; either absolute path or -## relative to current directory. If no `trainInstancesFiles` is provided, all -## the files in `trainInstancesDir` will be listed as instances. +## relative to current directory. If no `trainInstancesFiles` or +## `trainInstancesText` is provided, all the files in `trainInstancesDir` will +## be listed as instances. # trainInstancesDir = "./Instances" ## File that contains a list of training instances and optionally additional @@ -31,6 +40,11 @@ ## search for the files in this folder. # trainInstancesFile = "" +## A string that contains a list of training instances and optionally +## additional parameters for them. If `trainInstancesDir` is provided, `irace` +## will search for the files in this folder. +# trainInstancesText = "" + ## Randomly sample the training instances or use them in the order given. # sampleInstances = 1 @@ -42,6 +56,10 @@ ## parameters for them. # testInstancesFile = "" +## A string containing a list of test instances and optionally additional +## parameters for them. +# testInstancesText = "" + ## Number of elite configurations returned by irace that will be tested if ## test instances are provided. # testNbElites = 1 diff --git a/man/checkIraceScenario.Rd b/man/checkIraceScenario.Rd index ef48f6ed..15e7e719 100644 --- a/man/checkIraceScenario.Rd +++ b/man/checkIraceScenario.Rd @@ -25,9 +25,10 @@ settings provided and trying to run the target-algorithm. } \details{ If the \code{parameters} argument is missing, then the parameters -will be read from the file \code{parameterFile} given by \code{scenario}. If -\code{parameters} is provided, then \code{parameterFile} will not be read. This function will -try to execute the target-algorithm. +will be read from the file \code{parameterFile} or string \code{parameterText} +given by \code{scenario}. If \code{parameters} is provided, then \code{parameterFile} +or \code{parameterText} will not be read. This function will try to execute +the target-algorithm. } \seealso{ \describe{ diff --git a/man/defaultScenario.Rd b/man/defaultScenario.Rd index afcbdb43..823bffd5 100644 --- a/man/defaultScenario.Rd +++ b/man/defaultScenario.Rd @@ -51,6 +51,7 @@ The scenario list contains the following elements: \item Target algorithm parameters: \describe{ \item{\code{parameterFile}}{File that contains the description of the parameters of the target algorithm. (Default: \code{"./parameters.txt"})} +\item{\code{parameterText}}{A string that contains the description of the parameters of the target algorithm. (Default: \code{""})} \item{\code{digits}}{Maximum number of decimal places that are significant for numerical (real) parameters. (Default: \code{4})} } \item Target algorithm execution: @@ -72,13 +73,15 @@ The scenario list contains the following elements: \describe{ \item{\code{initConfigurations}}{Data frame describing initial configurations (usually read from a file using \code{readConfigurations}). (Default: \code{""})} \item{\code{configurationsFile}}{File that contains a table of initial configurations. If empty or \code{NULL}, all initial configurations are randomly generated. (Default: \code{""})} +\item{\code{configurationsText}}{A string that contains a table of initial configurations. If empty or \code{NULL}, all initial configurations are randomly generated. (Default: \code{""})} } \item Training instances: \describe{ \item{\code{instances}}{Character vector of the instances to be used in the \code{targetRunner}. (Default: \code{""})} -\item{\code{trainInstancesDir}}{Directory where training instances are located; either absolute path or relative to current directory. If no \code{trainInstancesFiles} is provided, all the files in \code{trainInstancesDir} will be listed as instances. (Default: \code{"./Instances"})} +\item{\code{trainInstancesDir}}{Directory where training instances are located; either absolute path or relative to current directory. If no \code{trainInstancesFiles} or \code{trainInstancesText} is provided, all the files in \code{trainInstancesDir} will be listed as instances. (Default: \code{"./Instances"})} \item{\code{trainInstancesFile}}{File that contains a list of training instances and optionally additional parameters for them. If \code{trainInstancesDir} is provided, \code{irace} will search for the files in this folder. (Default: \code{""})} -\item{\code{blockSize}}{Number of training instances, that make up a \verb{block' in \code{trainInstancesFile}. Elimination of configurations will only be performed after evaluating a complete block and never in the middle of a block. Each block typically contains one instance from each instance class (type or family). (Default: }1\verb{)\} \} \\item Tuning budget: \\describe\{ \\item\{}maxExperiments\verb{\}\{Maximum number of runs (invocations of \code{targetRunner}) that will be performed. It determines the maximum budget of experiments for the tuning. (Default: }0\verb{)\} \\item\{}maxTime\verb{\}\{Maximum total execution time in seconds for the executions of \code{targetRunner}. \code{targetRunner} must return two values: cost and time. (Default: }0\verb{)\} \\item\{}budgetEstimation\verb{\}\{Fraction (smaller than 1) of the budget used to estimate the mean computation time of a configuration. Only used when \code{maxTime} > 0 (Default: }0.02\verb{)\} \\item\{}minMeasurableTime\verb{\}\{Minimum time unit that is still (significantly) measureable. (Default: }0.01\verb{)\} \} \\item Statistical test: \\describe\{ \\item\{}testType\verb{\}\{Statistical test used for elimination. The default value selects \code{t-test} if \code{capping} is enabled or \code{F-test}, otherwise. Valid values are: F-test (Friedman test), t-test (pairwise t-tests with no correction), t-test-bonferroni (t-test with Bonferroni's correction for multiple comparisons), t-test-holm (t-test with Holm's correction for multiple comparisons). (Default: }""\verb{)\} \\item\{}firstTest\verb{\}\{Number of instances evaluated before the first elimination test. It must be a multiple of \code{eachTest}. (Default: }5\verb{)\} \\item\{}eachTest\verb{\}\{Number of instances evaluated between elimination tests. (Default: }1\verb{)\} \\item\{}confidence\verb{\}\{Confidence level for the elimination test. (Default: }0.95\verb{)\} \} \\item Adaptive capping: \\describe\{ \\item\{}capping\verb{\}\{Enable the use of adaptive capping, a technique designed for minimizing the computation time of configurations. Capping is enabled by default if \code{elitist} is active, \code{maxTime > 0} and \code{boundMax > 0}. (Default: }NA\verb{)\} \\item\{}cappingType\verb{\}\{Measure used to obtain the execution bound from the performance of the elite configurations.\\itemize\{\\item median: Median performance of the elite configurations.\\item mean: Mean performance of the elite configurations.\\item best: Best performance of the elite configurations.\\item worst: Worst performance of the elite configurations.\} (Default: }"median"\verb{)\} \\item\{}boundType\verb{\}\{Method to calculate the mean performance of elite configurations.\\itemize\{\\item candidate: Mean execution times across the executed instances and the current one.\\item instance: Execution time of the current instance.\} (Default: }"candidate"\verb{)\} \\item\{}boundMax\verb{\}\{Maximum execution bound for \code{targetRunner}. It must be specified when capping is enabled. (Default: }0\verb{)\} \\item\{}boundDigits\verb{\}\{Precision used for calculating the execution time. It must be specified when capping is enabled. (Default: }0\verb{)\} \\item\{}boundPar\verb{\}\{Penalization constant for timed out executions (executions that reach \code{boundMax} execution time). (Default: }1\verb{)\} \\item\{}boundAsTimeout\verb{\}\{Replace the configuration cost of bounded executions with \code{boundMax}. (Default: }1\verb{)\} \} \\item Recovery: \\describe\{ \\item\{}recoveryFile\verb{\}\{Previously saved log file to recover the execution of \code{irace}, either absolute path or relative to the current directory. If empty or \code{NULL}, recovery is not performed. (Default: }""\verb{)\} \} \\item Testing: \\describe\{ \\item\{}testInstancesDir\verb{\}\{Directory where testing instances are located, either absolute or relative to current directory. (Default: }""\verb{)\} \\item\{}testInstancesFile\verb{\}\{File containing a list of test instances and optionally additional parameters for them. (Default: }""\verb{)\} \\item\{}testInstances\verb{\}\{Character vector of the instances to be used in the \code{targetRunner} when executing the testing. (Default: }""\verb{)\} \\item\{}testNbElites\verb{\}\{Number of elite configurations returned by irace that will be tested if test instances are provided. (Default: }1\verb{)\} \\item\{}testIterationElites\verb{\}\{Enable/disable testing the elite configurations found at each iteration. (Default: }0`)} +\item{\code{trainInstancesText}}{A string that contains a list of training instances and optionally additional parameters for them. If \code{trainInstancesDir} is provided, \code{irace} will search for the files in this folder. (Default: \code{""})} +\item{\code{blockSize}}{Number of training instances, that make up a \verb{block' in \code{trainInstancesFile}. Elimination of configurations will only be performed after evaluating a complete block and never in the middle of a block. Each block typically contains one instance from each instance class (type or family). (Default: }1\verb{)\} \} \\item Tuning budget: \\describe\{ \\item\{}maxExperiments\verb{\}\{Maximum number of runs (invocations of \code{targetRunner}) that will be performed. It determines the maximum budget of experiments for the tuning. (Default: }0\verb{)\} \\item\{}maxTime\verb{\}\{Maximum total execution time in seconds for the executions of \code{targetRunner}. \code{targetRunner} must return two values: cost and time. (Default: }0\verb{)\} \\item\{}budgetEstimation\verb{\}\{Fraction (smaller than 1) of the budget used to estimate the mean computation time of a configuration. Only used when \code{maxTime} > 0 (Default: }0.02\verb{)\} \\item\{}minMeasurableTime\verb{\}\{Minimum time unit that is still (significantly) measureable. (Default: }0.01\verb{)\} \} \\item Statistical test: \\describe\{ \\item\{}testType\verb{\}\{Statistical test used for elimination. The default value selects \code{t-test} if \code{capping} is enabled or \code{F-test}, otherwise. Valid values are: F-test (Friedman test), t-test (pairwise t-tests with no correction), t-test-bonferroni (t-test with Bonferroni's correction for multiple comparisons), t-test-holm (t-test with Holm's correction for multiple comparisons). (Default: }""\verb{)\} \\item\{}firstTest\verb{\}\{Number of instances evaluated before the first elimination test. It must be a multiple of \code{eachTest}. (Default: }5\verb{)\} \\item\{}eachTest\verb{\}\{Number of instances evaluated between elimination tests. (Default: }1\verb{)\} \\item\{}confidence\verb{\}\{Confidence level for the elimination test. (Default: }0.95\verb{)\} \} \\item Adaptive capping: \\describe\{ \\item\{}capping\verb{\}\{Enable the use of adaptive capping, a technique designed for minimizing the computation time of configurations. Capping is enabled by default if \code{elitist} is active, \code{maxTime > 0} and \code{boundMax > 0}. (Default: }NA\verb{)\} \\item\{}cappingType\verb{\}\{Measure used to obtain the execution bound from the performance of the elite configurations.\\itemize\{\\item median: Median performance of the elite configurations.\\item mean: Mean performance of the elite configurations.\\item best: Best performance of the elite configurations.\\item worst: Worst performance of the elite configurations.\} (Default: }"median"\verb{)\} \\item\{}boundType\verb{\}\{Method to calculate the mean performance of elite configurations.\\itemize\{\\item candidate: Mean execution times across the executed instances and the current one.\\item instance: Execution time of the current instance.\} (Default: }"candidate"\verb{)\} \\item\{}boundMax\verb{\}\{Maximum execution bound for \code{targetRunner}. It must be specified when capping is enabled. (Default: }0\verb{)\} \\item\{}boundDigits\verb{\}\{Precision used for calculating the execution time. It must be specified when capping is enabled. (Default: }0\verb{)\} \\item\{}boundPar\verb{\}\{Penalization constant for timed out executions (executions that reach \code{boundMax} execution time). (Default: }1\verb{)\} \\item\{}boundAsTimeout\verb{\}\{Replace the configuration cost of bounded executions with \code{boundMax}. (Default: }1\verb{)\} \} \\item Recovery: \\describe\{ \\item\{}recoveryFile\verb{\}\{Previously saved log file to recover the execution of \code{irace}, either absolute path or relative to the current directory. If empty or \code{NULL}, recovery is not performed. (Default: }""\verb{)\} \} \\item Testing: \\describe\{ \\item\{}testInstancesDir\verb{\}\{Directory where testing instances are located, either absolute or relative to current directory. (Default: }""\verb{)\} \\item\{}testInstancesFile\verb{\}\{File containing a list of test instances and optionally additional parameters for them. (Default: }""\verb{)\} \\item\{}testInstancesText\verb{\}\{A string containing a list of test instances and optionally additional parameters for them. (Default: }""\verb{)\} \\item\{}testInstances\verb{\}\{Character vector of the instances to be used in the \code{targetRunner} when executing the testing. (Default: }""\verb{)\} \\item\{}testNbElites\verb{\}\{Number of elite configurations returned by irace that will be tested if test instances are provided. (Default: }1\verb{)\} \\item\{}testIterationElites\verb{\}\{Enable/disable testing the elite configurations found at each iteration. (Default: }0`)} } } } diff --git a/man/irace.cmdline.Rd b/man/irace.cmdline.Rd index 6771ab13..cb38b934 100644 --- a/man/irace.cmdline.Rd +++ b/man/irace.cmdline.Rd @@ -105,9 +105,14 @@ List of command-line options:\preformatted{-h,--help Show this he --exec-dir Directory where the programs will be run. Default: ./. -p,--parameter-file File that contains the description of the parameters of the target algorithm. Default: ./parameters.txt. + --parameter-text A string that contains the description of the + parameters of the target algorithm. --configurations-file File that contains a table of initial configurations. If empty or `NULL`, all initial configurations are randomly generated. + --configurations-text A string that contains a table of initial + configurations. If empty or `NULL`, all initial + configurations are randomly generated. -l,--log-file File to save tuning results as an R dataset, either absolute path or relative to execDir. Default: ./irace.Rdata. @@ -117,19 +122,25 @@ List of command-line options:\preformatted{-h,--help Show this he not performed. --train-instances-dir Directory where training instances are located; either absolute path or relative to current directory. - If no `trainInstancesFiles` is provided, all the files - in `trainInstancesDir` will be listed as instances. - Default: ./Instances. + If no `trainInstancesFiles` or `trainInstancesText` is + provided, all the files in `trainInstancesDir` will be + listed as instances. Default: ./Instances. --train-instances-file File that contains a list of training instances and optionally additional parameters for them. If `trainInstancesDir` is provided, `irace` will search for the files in this folder. + --train-instances-text A string that contains a list of training instances + and optionally additional parameters for them. If + `trainInstancesDir` is provided, `irace` will search + for the files in this folder. --sample-instances Randomly sample the training instances or use them in the order given. Default: 1. --test-instances-dir Directory where testing instances are located, either absolute or relative to current directory. --test-instances-file File containing a list of test instances and optionally additional parameters for them. + --test-instances-text A string containing a list of test instances and + optionally additional parameters for them. --test-num-elites Number of elite configurations returned by irace that will be tested if test instances are provided. Default: 1. diff --git a/man/irace.main.Rd b/man/irace.main.Rd index dcd9a947..45c3cbed 100644 --- a/man/irace.main.Rd +++ b/man/irace.main.Rd @@ -87,11 +87,12 @@ Higher-level interface to launch irace. } \details{ This function checks the correctness of the scenario, reads the -parameter space from \code{scenario$parameterFile}, invokes \code{\link[=irace]{irace()}}, -prints its results in various formatted ways, (optionally) calls -\code{\link[=psRace]{psRace()}} and, finally, evaluates the best configurations on the test -instances (if provided). If you want a lower-level interface that just -runs irace, please see function \code{\link[=irace]{irace()}}. +parameter space from \code{scenario$parameterFile} or +\code{scenario$parameterText}, invokes \code{\link[=irace]{irace()}}, prints its results in +various formatted ways, (optionally) calls \code{\link[=psRace]{psRace()}} and, finally, +evaluates the best configurations on the test instances (if provided). +If you want a lower-level interface that just runs irace, +please see function \code{\link[=irace]{irace()}}. } \seealso{ \code{\link[=irace.cmdline]{irace.cmdline()}} a higher-level command-line interface to diff --git a/scripts/irace_options.json b/scripts/irace_options.json index be80bd09..3b4164fd 100644 --- a/scripts/irace_options.json +++ b/scripts/irace_options.json @@ -80,6 +80,16 @@ "vignettes": "File that contains the description of the parameters of the target algorithm. See \\autoref{sec:target parameters}.", "section": "Target algorithm parameters" }, + { + "name": "parameterText", + "type": "p", + "short": "", + "long": "--parameter-text", + "default": "", + "description": "A string that contains the description of the parameters of the target algorithm.", + "vignettes": "A string that contains the description of the parameters of the target algorithm. See \\autoref{sec:target parameters}.", + "section": "Target algorithm parameters" + }, { "name": "initConfigurations", "type": "x", @@ -96,10 +106,20 @@ "short": "", "long": "--configurations-file", "default": "", - "vignettes": "File containing a table of initial configurations. If empty or \\code{NULL}, \\irace will not use initial configurations. See \\autoref{sec:initial}.\n\\begin{xwarningbox}\nThe provided configurations must not violate the constraints described in \\parameter{parameterFile} and \\parameter{forbiddenFile}.\n\\end{xwarningbox}", + "vignettes": "File containing a table of initial configurations. If empty or \\code{NULL}, \\irace will not use initial configurations. See \\autoref{sec:initial}.\n\\begin{xwarningbox}\nThe provided configurations must not violate the constraints described in \\parameter{parameterFile} or \\parameter{parameterText} and \\parameter{forbiddenFile}.\n\\end{xwarningbox}", "section": "Initial configurations", "description": "File that contains a table of initial configurations. If empty or \\code{NULL}, all initial configurations are randomly generated." }, + { + "name": "configurationsText", + "type": "p", + "short": "", + "long": "--configurations-text", + "default": "", + "vignettes": "A string containing a table of initial configurations. If empty or \\code{NULL}, \\irace will not use initial configurations. See \\autoref{sec:initial}.\n\\begin{xwarningbox}\nThe provided configurations must not violate the constraints described in \\parameter{parameterFile} or \\parameter{parameterText} and \\parameter{forbiddenFile}.\n\\end{xwarningbox}", + "section": "Initial configurations", + "description": "A string that contains a table of initial configurations. If empty or \\code{NULL}, all initial configurations are randomly generated." + }, { "name": "logFile", "type": "p", @@ -138,7 +158,7 @@ "default": ".\/Instances", "vignettes": "Directory where training instances are located; either absolute path or relative to current directory. See \\autoref{sec:training}.", "section": "Training instances", - "description": "Directory where training instances are located; either absolute path or relative to current directory. If no \\code{trainInstancesFiles} is provided, all the files in \\code{trainInstancesDir} will be listed as instances." + "description": "Directory where training instances are located; either absolute path or relative to current directory. If no \\code{trainInstancesFiles} or \\code{trainInstancesText} is provided, all the files in \\code{trainInstancesDir} will be listed as instances." }, { "name": "trainInstancesFile", @@ -150,6 +170,16 @@ "section": "Training instances", "description": "File that contains a list of training instances and optionally additional parameters for them. If \\code{trainInstancesDir} is provided, \\code{irace} will search for the files in this folder." }, + { + "name": "trainInstancesText", + "type": "p", + "short": "", + "long": "--train-instances-text", + "default": "", + "vignettes": "A string that contains a list of instances and optionally additional parameters for them. See \\autoref{sec:training}.\n\\begin{xwarningbox}\nThe list of instances in \\parameter{trainInstancesText} is interpreted as file-system paths relative to \\parameter{trainInstancesDir}. When using an absolute path or instances that are not files, set \\code{trainInstancesDir=\"\"}.\n\\end{xwarningbox}", + "section": "Training instances", + "description": "A string that contains a list of training instances and optionally additional parameters for them. If \\code{trainInstancesDir} is provided, \\code{irace} will search for the files in this folder." + }, { "name": "sampleInstances", "type": "b", @@ -157,7 +187,7 @@ "long": "--sample-instances", "default": 1, "description": "Randomly sample the training instances or use them in the order given.", - "vignettes": "Enable\/disable the sampling of the training instances. If the option \\parameter{sampleInstances} is disabled, the instances are used in the order provided in the \\parameter{trainInstancesFile} or in the order they are read from the \\parameter{trainInstancesDir} when\\parameter{trainInstancesFile} is not provided. For more information about training instances see \\autoref{sec:training}.", + "vignettes": "Enable\/disable the sampling of the training instances. If the option \\parameter{sampleInstances} is disabled, the instances are used in the order provided in the \\parameter{trainInstancesFile} or \\parameter{trainInstancesText} or in the order they are read from the \\parameter{trainInstancesDir} when\\parameter{trainInstancesFile} nor \\parameter{trainInstancesText} is provided. For more information about training instances see \\autoref{sec:training}.", "section": "Internal \\irace options" }, { @@ -180,6 +210,16 @@ "vignettes": "File containing a list of test instances and, optionally, additional parameters for them.", "section": "Testing" }, + { + "name": "testInstancesText", + "type": "p", + "short": "", + "long": "--test-instances-text", + "default": "", + "description": "A string containing a list of test instances and optionally additional parameters for them.", + "vignettes": "A string containing a list of test instances and, optionally, additional parameters for them.", + "section": "Testing" + }, { "name": "testInstances", "type": "x", diff --git a/vignettes/section/irace-options.tex b/vignettes/section/irace-options.tex index 30e7101c..e3666388 100644 --- a/vignettes/section/irace-options.tex +++ b/vignettes/section/irace-options.tex @@ -72,7 +72,7 @@ \subsection[Internal irace options]{Internal \irace options} \begin{description} \defparameter{sampleInstances}{sample-instances}{1}% -Enable/disable the sampling of the training instances. If the option \parameter{sampleInstances} is disabled, the instances are used in the order provided in the \parameter{trainInstancesFile} or in the order they are read from the \parameter{trainInstancesDir} when\parameter{trainInstancesFile} is not provided. For more information about training instances see \autoref{sec:training}. +Enable/disable the sampling of the training instances. If the option \parameter{sampleInstances} is disabled, the instances are used in the order provided in the \parameter{trainInstancesFile} or \parameter{trainInstancesText} or in the order they are read from the \parameter{trainInstancesDir} when\parameter{trainInstancesFile} nor \parameter{trainInstancesText} is provided. For more information about training instances see \autoref{sec:training}. \defparameter{softRestart}{soft-restart}{1}% Enable/disable the soft-restart strategy that avoids premature convergence of the probabilistic model. When a sampled configuration is \emph{similar} to its parent configuration, the probabilistic model of these configurations is soft restarted. The soft-restart mechanism is explained in the \irace paper~\citep{LopDubPerStuBir2016irace}. The similarity of categorical and ordinal parameters is given by the hamming distance, and the option \parameter{softRestartThreshold} defines the similarity of numerical parameters. @@ -104,6 +104,9 @@ \defparameter[p]{parameterFile}{parameter-file}{./parameters.txt}% File that contains the description of the parameters of the target algorithm. See \autoref{sec:target parameters}. + \defparameter{parameterText}{parameter-text}{}% +A string that contains the description of the parameters of the target algorithm. See \autoref{sec:target parameters}. + \defparameter{digits}{digits}{4}% Maximum number of decimal places that are significant for numerical (real) parameters. @@ -160,7 +163,13 @@ \defparameter{configurationsFile}{configurations-file}{}% File containing a table of initial configurations. If empty or \code{NULL}, \irace will not use initial configurations. See \autoref{sec:initial}. \begin{xwarningbox} -The provided configurations must not violate the constraints described in \parameter{parameterFile} and \parameter{forbiddenFile}. +The provided configurations must not violate the constraints described in \parameter{parameterFile} or \parameter{parameterText} and \parameter{forbiddenFile}. +\end{xwarningbox} + + \defparameter{configurationsText}{configurations-text}{}% +A string containing a table of initial configurations. If empty or \code{NULL}, \irace will not use initial configurations. See \autoref{sec:initial}. +\begin{xwarningbox} +The provided configurations must not violate the constraints described in \parameter{parameterFile} or \parameter{parameterText} and \parameter{forbiddenFile}. \end{xwarningbox} \end{description} @@ -174,6 +183,12 @@ File that contains a list of instances and optionally additional parameters for them. See \autoref{sec:training}. \begin{xwarningbox} The list of instances in \parameter{trainInstancesFile} is interpreted as file-system paths relative to \parameter{trainInstancesDir}. When using an absolute path or instances that are not files, set \code{trainInstancesDir=""}. +\end{xwarningbox} + + \defparameter{trainInstancesText}{train-instances-text}{}% +A string that contains a list of instances and optionally additional parameters for them. See \autoref{sec:training}. +\begin{xwarningbox} +The list of instances in \parameter{trainInstancesText} is interpreted as file-system paths relative to \parameter{trainInstancesDir}. When using an absolute path or instances that are not files, set \code{trainInstancesDir=""}. \end{xwarningbox} \end{description} @@ -279,6 +294,9 @@ \defparameter{testInstancesFile}{test-instances-file}{}% File containing a list of test instances and, optionally, additional parameters for them. + \defparameter{testInstancesText}{test-instances-text}{}% +A string containing a list of test instances and, optionally, additional parameters for them. + \defparameter{testNbElites}{test-num-elites}{1}% Number of elite configurations returned by irace that will be tested if test instances are provided. For more information about the testing, see \autoref{sec:testing}.