From ea06cfa2ceef58a4e132955da886dde0a56284e8 Mon Sep 17 00:00:00 2001 From: Valentin Churavy Date: Thu, 18 Jan 2018 17:05:22 -0500 Subject: [PATCH] add cputime and realtime and cleanup --- src/BenchmarkTools.jl | 12 +++- src/execution.jl | 16 ++--- src/groups.jl | 5 +- src/serialization.jl | 2 +- src/timers/timers.jl | 9 +-- src/timers/unix.jl | 6 +- src/trials.jl | 134 ++++++++++++++++++++++++----------------- test/ExecutionTests.jl | 2 +- test/GroupsTests.jl | 26 ++++---- test/TrialsTests.jl | 58 ++++++++++-------- 10 files changed, 159 insertions(+), 111 deletions(-) diff --git a/src/BenchmarkTools.jl b/src/BenchmarkTools.jl index 284f11a2..c2215653 100644 --- a/src/BenchmarkTools.jl +++ b/src/BenchmarkTools.jl @@ -10,7 +10,13 @@ if VERSION >= v"0.7.0-DEV.3052" using Printf end -const BENCHMARKTOOLS_VERSION = v"0.2.2" +const BENCHMARKTOOLS_VERSION = v"0.3.0" + +########## +# Timers # +########## + +include("timers/timers.jl") ############## # Parameters # @@ -26,7 +32,9 @@ export loadparams! include("trials.jl") -export gctime, +export realtime, + cputime, + gctime, memory, allocs, params, diff --git a/src/execution.jl b/src/execution.jl index fb8e60f0..054a3991 100644 --- a/src/execution.jl +++ b/src/execution.jl @@ -314,18 +314,18 @@ function generate_benchmark_definition(eval_module, out_vars, setup_vars, core, $(setup) __evals = __params.evals __gc_start = Base.gc_num() - __start_realtime = Timers.realtime() - __start_cputime = Timers.cputime() + __start_realtime = BenchmarkTools.Timers.realtime() + __start_cputime = BenchmarkTools.Timers.cputime() __return_val = $(invocation) for __iter in 2:__evals $(invocation) end - __sample_realtime = Timers.realtime() - __start_realtime - __sample_cputime = Timers.cputime() - __start_cputime + __sample_realtime = $BenchmarkTools.Timers.realtime() - __start_realtime + __sample_cputime = $BenchmarkTools.Timers.cputime() - __start_cputime __gcdiff = Base.GC_Diff(Base.gc_num(), __gc_start) $(teardown) - __realtime = max((__realsample_time / __evals) - __params.overhead, 0.001) - __cputime = max((__cpusample_time / __evals) - __params.overhead, 0.001) + __realtime = max((__sample_realtime / __evals) - __params.overhead, 0.001) + __cputime = max((__sample_cputime / __evals) - __params.overhead, 0.001) __gctime = max((__gcdiff.total_time / __evals) - __params.overhead, 0.0) __memory = Int(fld(__gcdiff.allocd, __evals)) __allocs = Int(fld(__gcdiff.malloc + __gcdiff.realloc + @@ -382,7 +382,7 @@ is the *minimum* elapsed time measured during the benchmark. macro belapsed(args...) b = Expr(:macrocall, Symbol("@benchmark"), map(esc, args)...) return esc(quote - $BenchmarkTools.time($BenchmarkTools.minimum($BenchmarkTools.@benchmark $(args...)))/1e9 + $BenchmarkTools.realtime($BenchmarkTools.minimum($BenchmarkTools.@benchmark $(args...)))/1e9 end) end @@ -412,7 +412,7 @@ macro btime(args...) $trialmin = $BenchmarkTools.minimum($trial) $trialallocs = $BenchmarkTools.allocs($trialmin) println(" ", - $BenchmarkTools.prettytime($BenchmarkTools.time($trialmin)), + $BenchmarkTools.prettytime($BenchmarkTools.realtime($trialmin)), " (", $trialallocs , " allocation", $trialallocs == 1 ? "" : "s", ": ", $BenchmarkTools.prettymemory($BenchmarkTools.memory($trialmin)), ")") diff --git a/src/groups.jl b/src/groups.jl index 1c3db32c..fcebcc1c 100644 --- a/src/groups.jl +++ b/src/groups.jl @@ -81,7 +81,10 @@ Base.median(group::BenchmarkGroup) = mapvals(median, group) Base.min(groups::BenchmarkGroup...) = mapvals(min, groups...) Base.max(groups::BenchmarkGroup...) = mapvals(max, groups...) -Base.time(group::BenchmarkGroup) = mapvals(time, group) +import Base: time +@deprecate time(group::BenchmarkGroup) realtime(group) +realtime(group::BenchmarkGroup) = mapvals(realtime, group) +cputime(group::BenchmarkGroup) = mapvals(cputime, group) gctime(group::BenchmarkGroup) = mapvals(gctime, group) memory(group::BenchmarkGroup) = mapvals(memory, group) allocs(group::BenchmarkGroup) = mapvals(allocs, group) diff --git a/src/serialization.jl b/src/serialization.jl index 4ea66dcf..f568017e 100644 --- a/src/serialization.jl +++ b/src/serialization.jl @@ -1,5 +1,5 @@ if VERSION >= v"0.7.0-DEV.2437" - using Base.Meta.parse + using Base.Meta: parse end const VERSIONS = Dict("Julia" => string(VERSION), diff --git a/src/timers/timers.jl b/src/timers/timers.jl index e41ad41f..c27cca88 100644 --- a/src/timers/timers.jl +++ b/src/timers/timers.jl @@ -3,6 +3,7 @@ Based upon https://github.com/google/benchmark =# module Timers +import Compat """ realtime() @@ -26,13 +27,13 @@ function _applever() return VersionNumber(readchomp(`sw_vers -productVersion`)) end -if is_apple() && _applever() < v"10.12.0" +if Compat.Sys.isapple() && _applever() < v"10.12.0" include("darwin.jl") -elseif is_unix() +elseif Compat.Sys.isunix() include("unix.jl") -elseif is_windows() +elseif Compat.Sys.iswindows() include("windows.jl") else - error("$(Sys.Kernel) is not supported please file an issue") + error("$(Sys.KERNEL) is not supported please file an issue") end end # module \ No newline at end of file diff --git a/src/timers/unix.jl b/src/timers/unix.jl index 818400d4..2507663a 100644 --- a/src/timers/unix.jl +++ b/src/timers/unix.jl @@ -7,18 +7,18 @@ maketime(ts) = ts.tv_sec * 1e9 + ts.tv_nsec # From bits/times.h on a Linux system # Check if those are the same on BSD -if is_linux() +if Compat.Sys.islinux() const CLOCK_MONOTONIC = Cint(1) const CLOCK_PROCESS_CPUTIME_ID = Cint(2) elseif Sys.KERNEL == :FreeBSD # atleast on FreeBSD 11.1 const CLOCK_MONOTONIC = Cint(4) const CLOCK_PROCESS_CPUTIME_ID = Cint(14) -elseif is_apple() # Version 10.12 required +elseif Compat.Sys.isapple() # Version 10.12 required const CLOCK_MONOTONIC = Cint(6) const CLOCK_PROCESS_CPUTIME_ID = Cint(12) else error(""" - BenchmarkTools doesn't currently support your system. + BenchmarkTools doesn't currently support your operating system. Please file an issue, your kernel is $(Sys.KERNEL) """) end diff --git a/src/trials.jl b/src/trials.jl index 0509e4c7..e1f7aacf 100644 --- a/src/trials.jl +++ b/src/trials.jl @@ -4,26 +4,28 @@ mutable struct Trial params::Parameters - times::Vector{Float64} + realtimes::Vector{Float64} #TODO: rename to realtimes + cputimes::Vector{Float64} gctimes::Vector{Float64} memory::Int allocs::Int end -Trial(params::Parameters) = Trial(params, Float64[], Float64[], typemax(Int), typemax(Int)) +Trial(params::Parameters) = Trial(params, Float64[], Float64[], Float64[], typemax(Int), typemax(Int)) @compat function Base.:(==)(a::Trial, b::Trial) return a.params == b.params && - a.times == b.times && + a.realtimes == b.realtimes && a.gctimes == b.gctimes && a.memory == b.memory && a.allocs == b.allocs end -Base.copy(t::Trial) = Trial(copy(t.params), copy(t.times), copy(t.gctimes), t.memory, t.allocs) +Base.copy(t::Trial) = Trial(copy(t.params), copy(t.realtimes), copy(t.cputimes), copy(t.gctimes), t.memory, t.allocs) -function Base.push!(t::Trial, time, gctime, memory, allocs) - push!(t.times, time) +function Base.push!(t::Trial, realtime, cputime, gctime, memory, allocs) + push!(t.realtimes, realtime) + push!(t.cputimes, cputime) push!(t.gctimes, gctime) memory < t.memory && (t.memory = memory) allocs < t.allocs && (t.allocs = allocs) @@ -31,26 +33,30 @@ function Base.push!(t::Trial, time, gctime, memory, allocs) end function Base.deleteat!(t::Trial, i) - deleteat!(t.times, i) + deleteat!(t.realtimes, i) deleteat!(t.gctimes, i) return t end -Base.length(t::Trial) = length(t.times) -Base.getindex(t::Trial, i::Number) = push!(Trial(t.params), t.times[i], t.gctimes[i], t.memory, t.allocs) -Base.getindex(t::Trial, i) = Trial(t.params, t.times[i], t.gctimes[i], t.memory, t.allocs) +Base.length(t::Trial) = length(t.realtimes) +Base.getindex(t::Trial, i::Number) = push!(Trial(t.params), t.realtimes[i], t.cputimes[i], t.gctimes[i], t.memory, t.allocs) +Base.getindex(t::Trial, i) = Trial(t.params, t.realtimes[i], t.cputimes[i], t.gctimes[i], t.memory, t.allocs) Base.endof(t::Trial) = length(t) function Base.sort!(t::Trial) - inds = sortperm(t.times) - t.times = t.times[inds] + inds = sortperm(t.realtimes) + t.realtimes = t.realtimes[inds] + t.cputimes = t.cputimes[inds] t.gctimes = t.gctimes[inds] return t end Base.sort(t::Trial) = sort!(copy(t)) -Base.time(t::Trial) = time(minimum(t)) +import Base: time +@deprecate time(t::Trial) realtime(t) +realtime(t::Trial) = realtime(minimum(t)) +cputime(t::Trial) = cputime(minimum(t)) gctime(t::Trial) = gctime(minimum(t)) memory(t::Trial) = t.memory allocs(t::Trial) = t.allocs @@ -66,7 +72,7 @@ function skewcutoff(values) return length(current_values) + 1 end -skewcutoff(t::Trial) = skewcutoff(t.times) +skewcutoff(t::Trial) = skewcutoff(t.realtimes) function rmskew!(t::Trial) sort!(t) @@ -88,19 +94,21 @@ trim(t::Trial, percentage = 0.1) = t[1:max(1, floor(Int, length(t) - (length(t) mutable struct TrialEstimate params::Parameters - time::Float64 + realtime::Float64 + cputime::Float64 gctime::Float64 memory::Int allocs::Int end -function TrialEstimate(trial::Trial, t, gct) - return TrialEstimate(params(trial), t, gct, memory(trial), allocs(trial)) +function TrialEstimate(trial::Trial, realtime, cputime, gctime) + return TrialEstimate(params(trial), realtime, cputime, gctime, memory(trial), allocs(trial)) end @compat function Base.:(==)(a::TrialEstimate, b::TrialEstimate) return a.params == b.params && - a.time == b.time && + a.realtime == b.realtime && + a.cputime == b.cputime && a.gctime == b.gctime && a.memory == b.memory && a.allocs == b.allocs @@ -109,21 +117,24 @@ end Base.copy(t::TrialEstimate) = TrialEstimate(copy(t.params), t.time, t.gctime, t.memory, t.allocs) function Base.minimum(trial::Trial) - i = indmin(trial.times) - return TrialEstimate(trial, trial.times[i], trial.gctimes[i]) + i = indmin(trial.realtimes) + return TrialEstimate(trial, trial.realtimes[i], trial.cputimes[i], trial.gctimes[i]) end function Base.maximum(trial::Trial) - i = indmax(trial.times) - return TrialEstimate(trial, trial.times[i], trial.gctimes[i]) + i = indmax(trial.realtimes) + return TrialEstimate(trial, trial.realtimes[i], trial.cputimes[i], trial.gctimes[i]) end -Base.median(trial::Trial) = TrialEstimate(trial, median(trial.times), median(trial.gctimes)) -Base.mean(trial::Trial) = TrialEstimate(trial, mean(trial.times), mean(trial.gctimes)) +Base.median(trial::Trial) = TrialEstimate(trial, median(trial.realtimes), median(trial.cputimes), median(trial.gctimes)) +Base.mean(trial::Trial) = TrialEstimate(trial, mean(trial.realtimes), mean(trial.cputimes), mean(trial.gctimes)) -Base.isless(a::TrialEstimate, b::TrialEstimate) = isless(time(a), time(b)) +Base.isless(a::TrialEstimate, b::TrialEstimate) = isless(realtime(a), realtime(b)) -Base.time(t::TrialEstimate) = t.time +import Base: time +@deprecate time(t::TrialEstimate) realtime(t) +realtime(t::TrialEstimate) = t.realtime +cputime(t::TrialEstimate) = t.cputime gctime(t::TrialEstimate) = t.gctime memory(t::TrialEstimate) = t.memory allocs(t::TrialEstimate) = t.allocs @@ -135,7 +146,8 @@ params(t::TrialEstimate) = t.params mutable struct TrialRatio params::Parameters - time::Float64 + realtime::Float64 + cputime::Float64 gctime::Float64 memory::Float64 allocs::Float64 @@ -143,15 +155,19 @@ end @compat function Base.:(==)(a::TrialRatio, b::TrialRatio) return a.params == b.params && - a.time == b.time && + a.realtime == b.realtime && + a.cputime == b.cputime && a.gctime == b.gctime && a.memory == b.memory && a.allocs == b.allocs end -Base.copy(t::TrialRatio) = TrialRatio(copy(t.params), t.time, t.gctime, t.memory, t.allocs) +Base.copy(t::TrialRatio) = TrialRatio(copy(t.params), t.realtime, t.cputime, t.gctime, t.memory, t.allocs) -Base.time(t::TrialRatio) = t.time +import Base: time +@deprecate time(t::TrialRatio) realtime(t) +realtime(t::TrialRatio) = t.realtime +cputime(t::TrialRatio) = t.cputime gctime(t::TrialRatio) = t.gctime memory(t::TrialRatio) = t.memory allocs(t::TrialRatio) = t.allocs @@ -168,11 +184,13 @@ function ratio(a::TrialEstimate, b::TrialEstimate) ttol = max(params(a).time_tolerance, params(b).time_tolerance) mtol = max(params(a).memory_tolerance, params(b).memory_tolerance) p = Parameters(params(a); time_tolerance = ttol, memory_tolerance = mtol) - return TrialRatio(p, ratio(time(a), time(b)), ratio(gctime(a), gctime(b)), - ratio(memory(a), memory(b)), ratio(allocs(a), allocs(b))) + return TrialRatio(p, ratio(realtime(a), realtime(b)), ratio(cputime(a), cputime(b)), + ratio(gctime(a), gctime(b)), ratio(memory(a), memory(b)), + ratio(allocs(a), allocs(b))) end -gcratio(t::TrialEstimate) = ratio(gctime(t), time(t)) +gcratio(t::TrialEstimate) = ratio(gctime(t), realtime(t)) +cpuratio(t::TrialEstimate) = ratio(cputime(t), realtime(t)) ################## # TrialJudgement # @@ -180,25 +198,30 @@ gcratio(t::TrialEstimate) = ratio(gctime(t), time(t)) struct TrialJudgement ratio::TrialRatio - time::Symbol + realtime::Symbol + cputime::Symbol memory::Symbol end function TrialJudgement(r::TrialRatio) ttol = params(r).time_tolerance mtol = params(r).memory_tolerance - return TrialJudgement(r, judge(time(r), ttol), judge(memory(r), mtol)) + return TrialJudgement(r, judge(realtime(r), ttol), judge(cputime(r), ttol), judge(memory(r), mtol)) end @compat function Base.:(==)(a::TrialJudgement, b::TrialJudgement) return a.ratio == b.ratio && - a.time == b.time && + a.realtime == b.realtime && + a.cputime == b.cputime && a.memory == b.memory end -Base.copy(t::TrialJudgement) = TrialJudgement(copy(t.params), t.time, t.memory) +Base.copy(t::TrialJudgement) = TrialJudgement(copy(t.params), t.realtime, t.cputime, t.memory) -Base.time(t::TrialJudgement) = t.time +import Base: time +@deprecate time(t::TrialJudgement) realtime(t) +realtime(t::TrialJudgement) = t.realtime +cputime(t::TrialJudgement) = t.cputime memory(t::TrialJudgement) = t.memory ratio(t::TrialJudgement) = t.ratio params(t::TrialJudgement) = params(ratio(t)) @@ -221,9 +244,9 @@ function judge(ratio::Real, tolerance::Float64) end end -isimprovement(t::TrialJudgement) = time(t) == :improvement || memory(t) == :improvement -isregression(t::TrialJudgement) = time(t) == :regression || memory(t) == :regression -isinvariant(t::TrialJudgement) = time(t) == :invariant && memory(t) == :invariant +isimprovement(t::TrialJudgement) = realtime(t) == :improvement || cputime(t) == :improvement || memory(t) == :improvement +isregression(t::TrialJudgement) = realtime(t) == :regression || cputime(t) == :regression || memory(t) == :regression +isinvariant(t::TrialJudgement) = realtime(t) == :invariant && cputime(t) == :invariant && memory(t) == :invariant ################### # Pretty Printing # @@ -262,10 +285,10 @@ function prettymemory(b) return string(@sprintf("%.2f", value), " ", units) end -Base.show(io::IO, t::Trial) = print(io, "Trial(", prettytime(time(t)), ")") -Base.show(io::IO, t::TrialEstimate) = print(io, "TrialEstimate(", prettytime(time(t)), ")") -Base.show(io::IO, t::TrialRatio) = print(io, "TrialRatio(", prettypercent(time(t)), ")") -Base.show(io::IO, t::TrialJudgement) = print(io, "TrialJudgement(", prettydiff(time(ratio(t))), " => ", time(t), ")") +Base.show(io::IO, t::Trial) = print(io, "Trial(", prettytime(realtime(t)), ")") +Base.show(io::IO, t::TrialEstimate) = print(io, "TrialEstimate(", prettytime(realtime(t)), ")") +Base.show(io::IO, t::TrialRatio) = print(io, "TrialRatio(", prettypercent(realtime(t)), ")") +Base.show(io::IO, t::TrialJudgement) = print(io, "TrialJudgement(", prettydiff(realtime(ratio(t))), " => ", realtime(t), ")") @compat function Base.show(io::IO, ::MIME"text/plain", t::Trial) if length(t) > 0 @@ -275,10 +298,10 @@ Base.show(io::IO, t::TrialJudgement) = print(io, "TrialJudgement(", prettydiff(t avg = mean(t) memorystr = string(prettymemory(memory(min))) allocsstr = string(allocs(min)) - minstr = string(prettytime(time(min)), " (", prettypercent(gcratio(min)), " GC)") - maxstr = string(prettytime(time(med)), " (", prettypercent(gcratio(med)), " GC)") - medstr = string(prettytime(time(avg)), " (", prettypercent(gcratio(avg)), " GC)") - meanstr = string(prettytime(time(max)), " (", prettypercent(gcratio(max)), " GC)") + minstr = string(prettytime(realtime(min)), " ( ", prettypercent(cpuratio(min)) ,"CPU) (", prettypercent(gcratio(min)), " GC)") + maxstr = string(prettytime(realtime(med)), " (", prettypercent(cpuratio(med)) ,"CPU) (", prettypercent(gcratio(med)), " GC)") + medstr = string(prettytime(realtime(avg)), " (", prettypercent(cpuratio(avg)) ,"CPU) (", prettypercent(gcratio(avg)), " GC)") + meanstr = string(prettytime(realtime(max)), " (", prettypercent(cpuratio(max)) ,"CPU) (", prettypercent(gcratio(max)), " GC)") else memorystr = "N/A" allocsstr = "N/A" @@ -302,15 +325,17 @@ end @compat function Base.show(io::IO, ::MIME"text/plain", t::TrialEstimate) println(io, "BenchmarkTools.TrialEstimate: ") - println(io, " time: ", prettytime(time(t))) - println(io, " gctime: ", prettytime(gctime(t)), " (", prettypercent(gctime(t) / time(t)),")") + println(io, " realtime: ", prettytime(realtime(t))) + println(io, " cputime: ", prettytime(cputime(t)), " (", prettypercent(cputime(t) / realtime(t)),")") + println(io, " gctime: ", prettytime(gctime(t)), " (", prettypercent(gctime(t) / realtime(t)),")") println(io, " memory: ", prettymemory(memory(t))) print(io, " allocs: ", allocs(t)) end @compat function Base.show(io::IO, ::MIME"text/plain", t::TrialRatio) println(io, "BenchmarkTools.TrialRatio: ") - println(io, " time: ", time(t)) + println(io, " realtime: ", realtime(t)) + println(io, " cputime: ", cputime(t)) println(io, " gctime: ", gctime(t)) println(io, " memory: ", memory(t)) print(io, " allocs: ", allocs(t)) @@ -318,6 +343,7 @@ end @compat function Base.show(io::IO, ::MIME"text/plain", t::TrialJudgement) println(io, "BenchmarkTools.TrialJudgement: ") - println(io, " time: ", prettydiff(time(ratio(t))), " => ", time(t), " (", prettypercent(params(t).time_tolerance), " tolerance)") - print(io, " memory: ", prettydiff(memory(ratio(t))), " => ", memory(t), " (", prettypercent(params(t).memory_tolerance), " tolerance)") + println(io, " realtime: ", prettydiff(realtime(ratio(t))), " => ", realtime(t), " (", prettypercent(params(t).time_tolerance), " tolerance)") + println(io, " cputime: ", prettydiff(cputime(ratio(t))), " => ", cputime(t), " (", prettypercent(params(t).time_tolerance), " tolerance)") + print(io, " memory: ", prettydiff(memory(ratio(t))), " => ", memory(t), " (", prettypercent(params(t).memory_tolerance), " tolerance)") end diff --git a/test/ExecutionTests.jl b/test/ExecutionTests.jl index d79f3f73..0d6840d9 100644 --- a/test/ExecutionTests.jl +++ b/test/ExecutionTests.jl @@ -147,7 +147,7 @@ tune!(b) # This test is volatile in nonquiescent environments (e.g. Travis) # BenchmarkTools.DEFAULT_PARAMETERS.overhead = BenchmarkTools.estimate_overhead() -# @test time(minimum(@benchmark nothing)) == 1 +# @test realtime(minimum(@benchmark nothing)) == 1 @test [:x, :y, :z, :v, :w] == BenchmarkTools.collectvars(quote x = 1 + 3 diff --git a/test/GroupsTests.jl b/test/GroupsTests.jl index c46f186a..574362fe 100644 --- a/test/GroupsTests.jl +++ b/test/GroupsTests.jl @@ -16,9 +16,9 @@ seteq(a, b) = length(a) == length(b) == length(intersect(a, b)) g1 = BenchmarkGroup(["1", "2"]) -t1a = TrialEstimate(Parameters(time_tolerance = .05, memory_tolerance = .05), 32, 1, 2, 3) -t1b = TrialEstimate(Parameters(time_tolerance = .40, memory_tolerance = .40), 4123, 123, 43, 9) -tc = TrialEstimate(Parameters(time_tolerance = 1.0, memory_tolerance = 1.0), 1, 1, 1, 1) +t1a = TrialEstimate(Parameters(time_tolerance = .05, memory_tolerance = .05), 32, 32, 1, 2, 3) +t1b = TrialEstimate(Parameters(time_tolerance = .40, memory_tolerance = .40), 4123, 4123, 123, 43, 9) +tc = TrialEstimate(Parameters(time_tolerance = 1.0, memory_tolerance = 1.0), 1, 1, 1, 1, 1) g1["a"] = t1a g1["b"] = t1b @@ -29,14 +29,14 @@ g1similar = similar(g1) g2 = BenchmarkGroup(["2", "3"]) -t2a = TrialEstimate(Parameters(time_tolerance = .05, memory_tolerance = .05), 323, 1, 2, 3) -t2b = TrialEstimate(Parameters(time_tolerance = .40, memory_tolerance = .40), 1002, 123, 43, 9) +t2a = TrialEstimate(Parameters(time_tolerance = .05, memory_tolerance = .05), 323, 323, 1, 2, 3) +t2b = TrialEstimate(Parameters(time_tolerance = .40, memory_tolerance = .40), 1002, 1002, 123, 43, 9) g2["a"] = t2a g2["b"] = t2b g2["c"] = tc -trial = BenchmarkTools.Trial(Parameters(), [1, 2, 5], [0, 1, 1], 3, 56) +trial = BenchmarkTools.Trial(Parameters(), [1, 2, 5], [1, 2, 5], [0, 1, 1], 3, 56) gtrial = BenchmarkGroup([], Dict("t" => trial)) @@ -63,7 +63,8 @@ gtrial = BenchmarkGroup([], Dict("t" => trial)) @test isempty(g1similar) @test g1similar.tags == g1.tags -@test time(g1).data == Dict("a" => time(t1a), "b" => time(t1b), "c" => time(tc)) +@test realtime(g1).data == Dict("a" => realtime(t1a), "b" => realtime(t1b), "c" => realtime(tc)) +@test cputime(g1).data == Dict("a" => cputime(t1a), "b" => cputime(t1b), "c" => cputime(tc)) @test gctime(g1).data == Dict("a" => gctime(t1a), "b" => gctime(t1b), "c" => gctime(tc)) @test memory(g1).data == Dict("a" => memory(t1a), "b" => memory(t1b), "c" => memory(tc)) @test allocs(g1).data == Dict("a" => allocs(t1a), "b" => allocs(t1b), "c" => allocs(tc)) @@ -107,8 +108,8 @@ groupsa = BenchmarkGroup() groupsa["g1"] = g1 groupsa["g2"] = g2 g3a = addgroup!(groupsa, "g3", ["3", "4"]) -g3a["c"] = TrialEstimate(Parameters(time_tolerance = .05, memory_tolerance = .05), 6341, 23, 41, 536) -g3a["d"] = TrialEstimate(Parameters(time_tolerance = .13, memory_tolerance = .13), 12341, 3013, 2, 150) +g3a["c"] = TrialEstimate(Parameters(time_tolerance = .05, memory_tolerance = .05), 6341, 6341, 23, 41, 536) +g3a["d"] = TrialEstimate(Parameters(time_tolerance = .13, memory_tolerance = .13), 12341, 12341, 3013, 2, 150) groups_copy = copy(groupsa) groups_similar = similar(groupsa) @@ -117,8 +118,8 @@ groupsb = BenchmarkGroup() groupsb["g1"] = g1 groupsb["g2"] = g2 g3b = addgroup!(groupsb, "g3", ["3", "4"]) -g3b["c"] = TrialEstimate(Parameters(time_tolerance = .05, memory_tolerance = .05), 1003, 23, 41, 536) -g3b["d"] = TrialEstimate(Parameters(time_tolerance = .23, memory_tolerance = .23), 25341, 3013, 2, 150) +g3b["c"] = TrialEstimate(Parameters(time_tolerance = .05, memory_tolerance = .05), 1003, 1003, 23, 41, 536) +g3b["d"] = TrialEstimate(Parameters(time_tolerance = .23, memory_tolerance = .23), 25341, 25341, 3013, 2, 150) groupstrial = BenchmarkGroup() groupstrial["g"] = gtrial @@ -126,7 +127,8 @@ groupstrial["g"] = gtrial # tests # #-------# -@test time(groupsa).data == Dict("g1" => time(g1), "g2" => time(g2), "g3" => time(g3a)) +@test realtime(groupsa).data == Dict("g1" => realtime(g1), "g2" => realtime(g2), "g3" => realtime(g3a)) +@test cputime(groupsa).data == Dict("g1" => cputime(g1), "g2" => cputime(g2), "g3" => cputime(g3a)) @test gctime(groupsa).data == Dict("g1" => gctime(g1), "g2" => gctime(g2), "g3" => gctime(g3a)) @test memory(groupsa).data == Dict("g1" => memory(g1), "g2" => memory(g2), "g3" => memory(g3a)) @test allocs(groupsa).data == Dict("g1" => allocs(g1), "g2" => allocs(g2), "g3" => allocs(g3a)) diff --git a/test/TrialsTests.jl b/test/TrialsTests.jl index 27cac295..dddffd26 100644 --- a/test/TrialsTests.jl +++ b/test/TrialsTests.jl @@ -9,14 +9,14 @@ using Compat.Test ######### trial1 = BenchmarkTools.Trial(BenchmarkTools.Parameters(evals = 2)) -push!(trial1, 2, 1, 4, 5) -push!(trial1, 21, 0, 41, 51) +push!(trial1, 2, 2, 1, 4, 5) +push!(trial1, 21, 21, 0, 41, 51) trial2 = BenchmarkTools.Trial(BenchmarkTools.Parameters(time_tolerance = 0.15)) -push!(trial2, 21, 0, 41, 51) -push!(trial2, 2, 1, 4, 5) +push!(trial2, 21, 21, 0, 41, 51) +push!(trial2, 2, 2, 1, 4, 5) -push!(trial2, 21, 0, 41, 51) +push!(trial2, 21, 21, 0, 41, 51) @test length(trial2) == 3 deleteat!(trial2, 3) @test length(trial1) == length(trial2) == 2 @@ -24,7 +24,8 @@ sort!(trial2) @test trial1.params == BenchmarkTools.Parameters(evals = trial1.params.evals) @test trial2.params == BenchmarkTools.Parameters(time_tolerance = trial2.params.time_tolerance) -@test trial1.times == trial2.times == [2.0, 21.0] +@test trial1.realtimes == trial2.realtimes == [2.0, 21.0] +@test trial1.cputimes == trial2.cputimes == [2.0, 21.0] @test trial1.gctimes == trial2.gctimes == [1.0, 0.0] @test trial1.memory == trial2.memory == 4 @test trial1.allocs == trial2.allocs == 5 @@ -33,18 +34,19 @@ trial2.params = trial1.params @test trial1 == trial2 -@test trial1[2] == push!(BenchmarkTools.Trial(BenchmarkTools.Parameters(evals = 2)), 21, 0, 4, 5) +@test trial1[2] == push!(BenchmarkTools.Trial(BenchmarkTools.Parameters(evals = 2)), 21, 21, 0, 4, 5) @test trial1[1:end] == trial1 -@test time(trial1) == time(trial2) == 2.0 -@test gctime(trial1) == gctime(trial2) == 1.0 -@test memory(trial1) == memory(trial2) == trial1.memory -@test allocs(trial1) == allocs(trial2) == trial1.allocs -@test params(trial1) == params(trial2) == trial1.params +@test realtime(trial1) == realtime(trial2) == 2.0 +@test cputime(trial1) == cputime(trial2) == 2.0 +@test gctime(trial1) == gctime(trial2) == 1.0 +@test memory(trial1) == memory(trial2) == trial1.memory +@test allocs(trial1) == allocs(trial2) == trial1.allocs +@test params(trial1) == params(trial2) == trial1.params # outlier trimming trial3 = BenchmarkTools.Trial(BenchmarkTools.Parameters(), [1, 2, 3, 10, 11], - [1, 1, 1, 1, 1], 1, 1) + [1, 2, 3, 10, 11], [1, 1, 1, 1, 1], 1, 1) trimtrial3 = rmskew(trial3) rmskew!(trial3) @@ -59,11 +61,11 @@ rmskew!(trial3) randtrial = BenchmarkTools.Trial(BenchmarkTools.Parameters()) for _ in 1:40 - push!(randtrial, rand(1:20), 1, 1, 1) + push!(randtrial, rand(1:20), 1, 1, 1, 1) end while mean(randtrial) <= median(randtrial) - push!(randtrial, rand(10:20), 1, 1, 1) + push!(randtrial, rand(10:20), 1, 1, 1, 1) end rmskew!(randtrial) @@ -73,7 +75,7 @@ tmed = median(randtrial) tmean = mean(randtrial) tmax = maximum(randtrial) -@test time(tmin) == time(randtrial) +@test realtime(tmin) == realtime(randtrial) @test gctime(tmin) == gctime(randtrial) @test memory(tmin) == memory(tmed) == memory(tmean) == memory(tmax) == memory(randtrial) @test allocs(tmin) == allocs(tmed) == allocs(tmean) == allocs(tmax) == allocs(randtrial) @@ -94,31 +96,36 @@ x, y = rand(randrange), rand(randrange) @test (ratio(x, x) == 1.0) && (ratio(y, y) == 1.0) @test ratio(0.0, 0.0) == 1.0 -ta = BenchmarkTools.TrialEstimate(BenchmarkTools.Parameters(), rand(), rand(), rand(Int), rand(Int)) -tb = BenchmarkTools.TrialEstimate(BenchmarkTools.Parameters(), rand(), rand(), rand(Int), rand(Int)) +ta = BenchmarkTools.TrialEstimate(BenchmarkTools.Parameters(), rand(), rand(), rand(), rand(Int), rand(Int)) +tb = BenchmarkTools.TrialEstimate(BenchmarkTools.Parameters(), rand(), rand(), rand(), rand(Int), rand(Int)) tr = ratio(ta, tb) -@test time(tr) == ratio(time(ta), time(tb)) +@test realtime(tr) == ratio(realtime(ta), realtime(tb)) +@test cputime(tr) == ratio(cputime(ta), cputime(tb)) @test gctime(tr) == ratio(gctime(ta), gctime(tb)) @test memory(tr) == ratio(memory(ta), memory(tb)) @test allocs(tr) == ratio(allocs(ta), allocs(tb)) @test params(tr) == params(ta) == params(tb) -@test BenchmarkTools.gcratio(ta) == ratio(gctime(ta), time(ta)) -@test BenchmarkTools.gcratio(tb) == ratio(gctime(tb), time(tb)) +@test BenchmarkTools.gcratio(ta) == ratio(gctime(ta), realtime(ta)) +@test BenchmarkTools.gcratio(tb) == ratio(gctime(tb), realtime(tb)) + +@test BenchmarkTools.cpuratio(ta) == ratio(cputime(ta), realtime(ta)) +@test BenchmarkTools.cpuratio(tb) == ratio(cputime(tb), realtime(tb)) ################## # TrialJudgement # ################## -ta = BenchmarkTools.TrialEstimate(BenchmarkTools.Parameters(time_tolerance = 0.50, memory_tolerance = 0.50), 0.49, 0.0, 2, 1) -tb = BenchmarkTools.TrialEstimate(BenchmarkTools.Parameters(time_tolerance = 0.05, memory_tolerance = 0.05), 1.00, 0.0, 1, 1) +ta = BenchmarkTools.TrialEstimate(BenchmarkTools.Parameters(time_tolerance = 0.50, memory_tolerance = 0.50), 0.49, 0.49, 0.0, 2, 1) +tb = BenchmarkTools.TrialEstimate(BenchmarkTools.Parameters(time_tolerance = 0.05, memory_tolerance = 0.05), 1.00, 1.00, 0.0, 1, 1) tr = ratio(ta, tb) tj_ab = judge(ta, tb) tj_r = judge(tr) @test ratio(tj_ab) == ratio(tj_r) == tr -@test time(tj_ab) == time(tj_r) == :improvement +@test realtime(tj_ab) == realtime(tj_r) == :improvement +@test cputime(tj_ab) == cputime(tj_r) == :improvement @test memory(tj_ab) == memory(tj_r) == :regression @test tj_ab == tj_r @@ -127,7 +134,8 @@ tj_r_2 = judge(tr; time_tolerance = 2.0, memory_tolerance = 2.0) @test tj_ab_2 == tj_r_2 @test ratio(tj_ab_2) == ratio(tj_r_2) -@test time(tj_ab_2) == time(tj_r_2) == :invariant +@test realtime(tj_ab_2) == realtime(tj_r_2) == :invariant +@test cputime(tj_ab_2) == cputime(tj_r_2) == :invariant @test memory(tj_ab_2) == memory(tj_r_2) == :invariant @test !(isinvariant(tj_ab))