Skip to content

Commit

Permalink
Basic reference_job feature
Browse files Browse the repository at this point in the history
Basis is from #281 which we let lie for too long, sorry.

Also too much had changed in the mean time (`RelativeStatistics`)
so reopening it didn't make a ton of sesnse.

Fixes #179
  • Loading branch information
PragTob committed Jul 7, 2024
1 parent a62708c commit aee34d3
Show file tree
Hide file tree
Showing 3 changed files with 127 additions and 12 deletions.
7 changes: 5 additions & 2 deletions lib/benchee/configuration.ex
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,8 @@ defmodule Benchee.Configuration do
after_scenario: nil,
measure_function_call_overhead: false,
title: nil,
profile_after: false
profile_after: false,
reference_job: nil

@typedoc """
The configuration supplied by the user as either a map or a keyword list
Expand Down Expand Up @@ -133,6 +134,7 @@ defmodule Benchee.Configuration do
[`:cprof`](https://hexdocs.pm/mix/Mix.Tasks.Profile.Cprof.html),
[`:eprof`](https://hexdocs.pm/mix/Mix.Tasks.Profile.Eprof.html) and
[`:fprof`](https://hexdocs.pm/mix/Mix.Tasks.Profile.Fprof.html).
* `:reference_job` - Set which function in the benchmark all others will be compared against.
"""
@type user_configuration :: map | keyword

Expand Down Expand Up @@ -163,7 +165,8 @@ defmodule Benchee.Configuration do
after_scenario: Hooks.hook_function() | nil,
measure_function_call_overhead: boolean,
title: String.t() | nil,
profile_after: boolean | atom | {atom, keyword}
profile_after: boolean | atom | {atom, keyword},
reference_job: String.t() | nil
}

@time_keys [:time, :warmup, :memory_time, :reduction_time]
Expand Down
32 changes: 24 additions & 8 deletions lib/benchee/relative_statistics.ex
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ defmodule Benchee.RelativeStatistics do
has to happen before they are loaded to avoid recalculating their statistics.
"""

alias Benchee.{Scenario, Statistics, Suite}
alias Benchee.{Configuration, Scenario, Statistics, Suite}

@doc """
Calculate the statistics of scenarios relative to each other and sorts scenarios.
Expand All @@ -22,17 +22,17 @@ defmodule Benchee.RelativeStatistics do
"""
@spec relative_statistics(Suite.t()) :: Suite.t()
def relative_statistics(suite) do
%Suite{suite | scenarios: calculate_relative_statistics(suite.scenarios)}
%Suite{suite | scenarios: calculate_relative_statistics(suite.scenarios, suite.configuration)}
end

defp calculate_relative_statistics([]), do: []
defp calculate_relative_statistics([], _config), do: []

defp calculate_relative_statistics(scenarios) do
defp calculate_relative_statistics(scenarios, config) do
scenarios
|> scenarios_by_input()
|> Enum.flat_map(fn scenarios_with_same_input ->
sorted_scenarios = sort(scenarios_with_same_input)
{reference, others} = split_reference_scenario(sorted_scenarios)
{reference, others} = split_reference_scenario(sorted_scenarios, config)
others_with_relative = statistics_relative_to(others, reference)
[reference | others_with_relative]
end)
Expand All @@ -59,9 +59,25 @@ defmodule Benchee.RelativeStatistics do
end)
end

# right now we take the first scenario as we sorted them and it is the fastest,
# whenever we implement #179 though this becomes more involved
defp split_reference_scenario(scenarios) do
defp split_reference_scenario(scenarios, config)

defp split_reference_scenario(scenarios, %Configuration{reference_job: reference_job})
when is_binary(reference_job) do
split_scenarios =
Enum.split_with(scenarios, fn scenario -> scenario.name == reference_job end)

case split_scenarios do
{[reference_scenario], others} -> {reference_scenario, others}
_reference_not_found -> fastest_as_reference(scenarios)
end
end

# no reference_job configured? Just take the first one, as this is post sort it'll be the fastet
defp split_reference_scenario(scenarios, _config) do
fastest_as_reference(scenarios)
end

defp fastest_as_reference(scenarios) do
[reference | others] = scenarios
{reference, others}
end
Expand Down
100 changes: 98 additions & 2 deletions test/benchee/relative_statistics_test.exs
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
defmodule Benchee.RelativeStatistcsTest do
use ExUnit.Case, async: true

alias Benchee.{CollectionData, Scenario, Statistics, Suite}
alias Benchee.{CollectionData, Configuration, Scenario, Statistics, Suite}
import Benchee.RelativeStatistics

describe "computing relative statistics" do
Expand Down Expand Up @@ -43,7 +43,7 @@ defmodule Benchee.RelativeStatistcsTest do
assert stats.relative_less == nil
end

test "handles the fastest value being zerio alright" do
test "handles the fastest value being zero alright" do
suite = %Suite{
scenarios: [
scenario_with_average(0.0),
Expand Down Expand Up @@ -98,6 +98,55 @@ defmodule Benchee.RelativeStatistcsTest do
assert stats2.relative_less == 1.0
assert stats2.absolute_difference == 0.0
end

test "setting the reference job to be the slowest" do
suite = %Suite{
scenarios: [
named_scenario_with_average("A", 394.0, nil),
named_scenario_with_average("B", 14.0, nil)
],
configuration: %Configuration{reference_job: "A"}
}

suite = relative_statistics(suite)
# reference scenario always comes out on top even if slowest
assert [a_stats, b_stats] = stats_from(suite)

assert a_stats.relative_more == nil
assert a_stats.relative_less == nil
assert a_stats.absolute_difference == nil

assert b_stats.absolute_difference == -380.0
assert_in_delta b_stats.relative_more, 0.0355, 0.001
assert_in_delta b_stats.relative_less, 28.14, 0.1
end

test "sandwich reference job" do
suite = %Suite{
scenarios: [
named_scenario_with_average("A", 1.0, nil),
named_scenario_with_average("B", 2.0, nil),
named_scenario_with_average("C", 3.0, nil)
],
configuration: %Configuration{reference_job: "B"}
}

suite = relative_statistics(suite)
# reference scenario always comes out on top even if slowest
assert [b_stats, a_stats, c_stats] = stats_from(suite)

assert b_stats.relative_more == nil
assert b_stats.relative_less == nil
assert b_stats.absolute_difference == nil

assert a_stats.absolute_difference == -1.0
assert_in_delta a_stats.relative_more, 0.5, 0.001
assert_in_delta a_stats.relative_less, 2.0, 0.01

assert c_stats.absolute_difference == 1.0
assert_in_delta c_stats.relative_more, 1.5, 0.001
assert_in_delta c_stats.relative_less, 0.66, 0.01
end
end

describe "sorting behaviour" do
Expand Down Expand Up @@ -138,6 +187,24 @@ defmodule Benchee.RelativeStatistcsTest do

assert Enum.map(sorted, fn scenario -> scenario.name end) == ["1", "2", "3"]
end

test "given a configured reference_job returns that as the first regardless of performance" do
fourth = named_scenario_with_average("4", 400.1, nil)
second = named_scenario_with_average("2", 200.0, nil)
third = named_scenario_with_average("3", 400.0, nil)
first = named_scenario_with_average("1", 100.0, nil)
reference = named_scenario_with_average("Ref", 500.0, nil)

scenarios = [fourth, second, third, first, reference]

sorted =
relative_statistics(%Suite{
scenarios: scenarios,
configuration: %Configuration{reference_job: "Ref"}
}).scenarios

assert Enum.map(sorted, fn scenario -> scenario.name end) == ["Ref", "1", "2", "3", "4"]
end
end

describe "dealing correctly with different inputs" do
Expand Down Expand Up @@ -187,6 +254,35 @@ defmodule Benchee.RelativeStatistcsTest do
]
end

test "calculate the correct relatives based on input names even given a reference_job" do
suite = %Suite{
scenarios: [
named_input_scenario_with_average("A", "1", 100.0),
named_input_scenario_with_average("B", "1", 200.0),
named_input_scenario_with_average("A", "10", 1_000.0),
named_input_scenario_with_average("B", "10", 10_000.0)
],
configuration: %Configuration{reference_job: "B"}
}

suite = relative_statistics(suite)

stats =
suite.scenarios
|> Enum.map(
&{&1.name, &1.input_name, &1.run_time_data.statistics.absolute_difference,
&1.run_time_data.statistics.relative_more}
)
|> Enum.sort()

assert stats == [
{"A", "1", -100.0, 0.5},
{"A", "10", -9_000, 0.1},
{"B", "1", nil, nil},
{"B", "10", nil, nil}
]
end

test "calculate the correct relatives based on different ordering" do
suite = %Suite{
scenarios: [
Expand Down

0 comments on commit aee34d3

Please sign in to comment.