Merge pull request #216 from JuliaAI/dev

For a 0.8.5 release

Project.toml

@@ -1,7 +1,7 @@
 name = "MLJTuning"
 uuid = "03970b2e-30c4-11ea-3135-d1576263f10f"
 authors = ["Anthony D. Blaom <[email protected]>"]
-version = "0.8.4"
+version = "0.8.5"
 
 [deps]
 ComputationalResources = "ed09eef8-17a6-5b46-8889-db040fac31e3"
@@ -18,7 +18,7 @@ StatisticalMeasuresBase = "c062fc1d-0d66-479b-b6ac-8b44719de4cc"
 ComputationalResources = "0.3"
 Distributions = "0.22,0.23,0.24, 0.25"
 LatinHypercubeSampling = "1.7.2"
-MLJBase = "1"
+MLJBase = "1.3"
 ProgressMeter = "1.7.1"
 RecipesBase = "0.8,0.9,1"
 StatisticalMeasuresBase = "0.1.1"

src/tuned_models.jl

@@ -50,6 +50,7 @@ mutable struct DeterministicTunedModel{T,M<:DeterministicTypes} <: MLJBase.Deter
  acceleration_resampling::AbstractResource
  check_measure::Bool
  cache::Bool
+ compact_history::Bool
 end
 
 mutable struct ProbabilisticTunedModel{T,M<:ProbabilisticTypes} <: MLJBase.Probabilistic
@@ -69,6 +70,7 @@ mutable struct ProbabilisticTunedModel{T,M<:ProbabilisticTypes} <: MLJBase.Proba
  acceleration_resampling::AbstractResource
  check_measure::Bool
  cache::Bool
+ compact_history::Bool
 end
 
 const EitherTunedModel{T,M} =
@@ -176,6 +178,15 @@ key | value
 
 plus other key/value pairs specific to the `tuning` strategy.
 
+Each element of `history` is a property-accessible object with these properties:
+
+key | value
+--------------------|--------------------------------------------------
+`measure` | vector of measures (metrics)
+`measurement` | vector of measurements, one per measure
+`per_fold` | vector of vectors of unaggregated per-fold measurements
+`evaluation` | full `PerformanceEvaluation`/`CompactPerformaceEvaluation` object
+
 ### Complete list of key-word options
 
 - `model`: `Supervised` model prototype that is cloned and mutated to
@@ -240,27 +251,35 @@ plus other key/value pairs specific to the `tuning` strategy.
  user-suplied data; set to `false` to conserve memory. Speed gains
  likely limited to the case `resampling isa Holdout`.
 
+- `compact_history=true`: whether to write `CompactPerformanceEvaluation`](@ref) or
+ regular [`PerformanceEvaluation`](@ref) objects to the history (accessed via the
+ `:evaluation` key); the compact form excludes some fields to conserve memory.
+
 """
-function TunedModel(args...; model=nothing,
- models=nothing,
- tuning=nothing,
- resampling=MLJBase.Holdout(),
- measures=nothing,
- measure=measures,
- weights=nothing,
- class_weights=nothing,
- operations=nothing,
- operation=operations,
- ranges=nothing,
- range=ranges,
- selection_heuristic=NaiveSelection(),
- train_best=true,
- repeats=1,
- n=nothing,
- acceleration=default_resource(),
- acceleration_resampling=CPU1(),
- check_measure=true,
- cache=true)
+function TunedModel(
+ args...;
+ model=nothing,
+ models=nothing,
+ tuning=nothing,
+ resampling=MLJBase.Holdout(),
+ measures=nothing,
+ measure=measures,
+ weights=nothing,
+ class_weights=nothing,
+ operations=nothing,
+ operation=operations,
+ ranges=nothing,
+ range=ranges,
+ selection_heuristic=NaiveSelection(),
+ train_best=true,
+ repeats=1,
+ n=nothing,
+ acceleration=default_resource(),
+ acceleration_resampling=CPU1(),
+ check_measure=true,
+ cache=true,
+ compact_history=true,
+ )
 
  # user can specify model as argument instead of kwarg:
  length(args) < 2 || throw(ERR_TOO_MANY_ARGUMENTS)
@@ -339,7 +358,8 @@ function TunedModel(args...; model=nothing,
  acceleration,
  acceleration_resampling,
  check_measure,
- cache
+ cache,
+ compact_history,
  )
 
  if M <: DeterministicTypes
@@ -582,9 +602,10 @@ function assemble_events!(metamodels,
  check_measure = resampling_machine.model.check_measure,
  repeats = resampling_machine.model.repeats,
  acceleration = resampling_machine.model.acceleration,
- cache = resampling_machine.model.cache),
- resampling_machine.args...; cache=false) for
- _ in 2:length(partitions)]...]
+ cache = resampling_machine.model.cache,
+ compact = resampling_machine.model.compact
+ ), resampling_machine.args...; cache=false) for
+ _ in 2:length(partitions)]...]
 
  @sync for (i, parts) in enumerate(partitions)
  Threads.@spawn begin
@@ -736,21 +757,23 @@ function MLJBase.fit(tuned_model::EitherTunedModel{T,M},
 
  # instantiate resampler (`model` to be replaced with mutated
  # clones during iteration below):
- resampler = Resampler(model=model,
- resampling = deepcopy(tuned_model.resampling),
- measure = tuned_model.measure,
- weights = tuned_model.weights,
- class_weights = tuned_model.class_weights,
- operation = tuned_model.operation,
- check_measure = tuned_model.check_measure,
- repeats = tuned_model.repeats,
- acceleration = tuned_model.acceleration_resampling,
- cache = tuned_model.cache)
+ resampler = Resampler(
+ model=model,
+ resampling = deepcopy(tuned_model.resampling),
+ measure = tuned_model.measure,
+ weights = tuned_model.weights,
+ class_weights = tuned_model.class_weights,
+ operation = tuned_model.operation,
+ check_measure = tuned_model.check_measure,
+ repeats = tuned_model.repeats,
+ acceleration = tuned_model.acceleration_resampling,
+ cache = tuned_model.cache,
+ compact = tuned_model.compact_history,
+ )
  resampling_machine = machine(resampler, data...; cache=false)
  history, state = build!(nothing, n, tuning, model, model_buffer, state,
  verbosity, acceleration, resampling_machine)
 
-
  return finalize(
  tuned_model,
  model_buffer,
@@ -867,9 +890,9 @@ function MLJBase.reports_feature_importances(model::EitherTunedModel)
 end # This is needed in some cases (e.g tuning a `Pipeline`)
 
 function MLJBase.feature_importances(::EitherTunedModel, fitresult, report)
- # fitresult here is a machine created using the best_model obtained 
+ # fitresult here is a machine created using the best_model obtained
  # from the tuning process.
- # The line below will return `nothing` when the model being tuned doesn't 
+ # The line below will return `nothing` when the model being tuned doesn't
  # support feature_importances.
  return MLJBase.feature_importances(fitresult)
 end

test/tuned_models.jl

@@ -13,7 +13,7 @@ using Random
 Random.seed!(1234*myid())
 using .TestUtilities
 
-begin 
+begin
  N = 30
  x1 = rand(N);
  x2 = rand(N);
@@ -157,14 +157,14 @@ end
 
 @testset_accelerated "Feature Importances" accel begin
  # the DecisionTreeClassifier in /test/_models/ supports feature importances.
- tm0 = TunedModel( 
- model = trees[1], 
- measure = rms, 
- tuning = Grid(), 
- resampling = CV(nfolds = 5), 
- range = range( 
- trees[1], :max_depth, values = 1:10 
- ) 
+ tm0 = TunedModel(
+ model = trees[1],
+ measure = rms,
+ tuning = Grid(),
+ resampling = CV(nfolds = 5),
+ range = range(
+ trees[1], :max_depth, values = 1:10
+ )
  )
  @test reports_feature_importances(typeof(tm0))
  tm = TunedModel(
@@ -435,7 +435,7 @@ end
  model = DecisionTreeClassifier()
  tmodel = TunedModel(models=[model,])
  mach = machine(tmodel, X, y)
- @test mach isa Machine{<:Any,false}
+ @test !MLJBase.caches_data(mach)
  fit!(mach, verbosity=-1)
  @test !isdefined(mach, :data)
  MLJBase.Tables.istable(mach.cache[end].fitresult.machine.data[1])
@@ -490,7 +490,7 @@ end
  @test MLJBase.predict(mach2, (; x = rand(2))) ≈ fill(42.0, 2)
 end
 
-@testset_accelerated "full evaluation object" accel begin
+@testset_accelerated "evaluation object" accel begin
  X, y = make_regression(100, 2)
  dcr = DeterministicConstantRegressor()
 
@@ -504,10 +504,24 @@ end
  fit!(homach, verbosity=0);
  horep = report(homach)
  evaluations = getproperty.(horep.history, :evaluation)
+ @test first(evaluations) isa MLJBase.CompactPerformanceEvaluation
  measurements = getproperty.(evaluations, :measurement)
  models = getproperty.(evaluations, :model)
  @test all(==(measurements[1]), measurements)
  @test all(==(dcr), models)
+
+ homodel = TunedModel(
+ models=fill(dcr, 10),
+ resampling=Holdout(rng=StableRNG(1234)),
+ acceleration_resampling=accel,
+ measure=mae,
+ compact_history=false,
+ )
+ homach = machine(homodel, X, y)
+ fit!(homach, verbosity=0);
+ horep = report(homach)
+ evaluations = getproperty.(horep.history, :evaluation)
+ @test first(evaluations) isa MLJBase.PerformanceEvaluation
 end
 
 true