Skip to content

Commit

Permalink
Introduce Phd
Browse files Browse the repository at this point in the history
  • Loading branch information
AndreyAkinshin committed Aug 26, 2024
1 parent 8d829c2 commit 5692c7e
Show file tree
Hide file tree
Showing 76 changed files with 1,722 additions and 70 deletions.
2 changes: 1 addition & 1 deletion src/Perfolizer/Perfolizer.Demo/QuantileEstimatorDemo.cs
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ public void Run()
// * Akinshin, Andrey. "Trimmed Harrell-Davis quantile estimator based on the highest density interval of the given width."
// arXiv preprint arXiv:2111.11776 (2021).
// https://arxiv.org/abs/2111.11776
TrimmedHarrellDavisQuantileEstimator.SqrtInstance,
TrimmedHarrellDavisQuantileEstimator.Sqrt,
// The Sfakianakis-Verginis quantile estimators
// They use a weighted sum of all sample elements, weights are assigned according to the Binomial distribution
// * Sfakianakis, Michael E., and Dimitris G. Verginis. "A new family of nonparametric quantile estimators."
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ static SmokeQuantileEstimatorTests()
("HF8", new HyndmanFanQuantileEstimator(HyndmanFanType.Type8)),
("HF9", new HyndmanFanQuantileEstimator(HyndmanFanType.Type9)),
("HD", HarrellDavisQuantileEstimator.Instance),
("THD", TrimmedHarrellDavisQuantileEstimator.SqrtInstance),
("THD", TrimmedHarrellDavisQuantileEstimator.Sqrt),
("SV1", SfakianakisVerginis1QuantileEstimator.Instance),
("SV2", SfakianakisVerginis2QuantileEstimator.Instance),
("SV3", SfakianakisVerginis3QuantileEstimator.Instance),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ public void EstimationTest01()
{
var sample = new Sample(new double[] { -3, -2, -1, 0, 1, 2, 3 });
var probabilities = new Probability[] { 0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0 };
double[] actualQuantiles = TrimmedHarrellDavisQuantileEstimator.SqrtInstance.Quantiles(sample, probabilities);
double[] actualQuantiles = TrimmedHarrellDavisQuantileEstimator.Sqrt.Quantiles(sample, probabilities);
double[] expectedQuantiles =
{
-3, -2.72276083590394, -2.30045481668633, -1.66479731161074, -0.877210708467137, 2.22044604925031e-16, 0.877210708467138,
Expand All @@ -43,7 +43,7 @@ public void EstimationTest02()
{
var sample = new Sample(new ArithmeticProgressionSequence(0, 1).GenerateArray(100));
var probabilities = new Probability[] { 0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0 };
double[] actualQuantiles = TrimmedHarrellDavisQuantileEstimator.SqrtInstance.Quantiles(sample, probabilities);
double[] actualQuantiles = TrimmedHarrellDavisQuantileEstimator.Sqrt.Quantiles(sample, probabilities);
double[] expectedQuantiles =
{
0, 9.23049888501009, 19.1571295075902, 29.235253395819, 39.3599642036428, 49.5, 59.6400357963572, 69.764746604181,
Expand All @@ -57,7 +57,7 @@ public void EstimationTest03()
{
var sample = new Sample(new ArithmeticProgressionSequence(0, 1).GenerateArray(1_000_000));
var probabilities = new Probability[] { 0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0 };
double[] actualQuantiles = TrimmedHarrellDavisQuantileEstimator.SqrtInstance.Quantiles(sample, probabilities);
double[] actualQuantiles = TrimmedHarrellDavisQuantileEstimator.Sqrt.Quantiles(sample, probabilities);
double[] expectedQuantiles =
{
0, 99999.2066949439, 199999.152626934, 299999.235056257, 399999.360305779, 499999.5, 599999.639694222, 699999.764943743,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ public class MetrologyTests
[InlineData("20GHz")]
public void MeasurementUnitToStringParseTest(string s)
{
if (!MeasurementValue.TryParse(s, out var value))
if (!Measurement.TryParse(s, out var value))
throw new Exception($"Failed to parse '{s}'");
Assert.Equal(s, value.ToString());
}
Expand Down
21 changes: 16 additions & 5 deletions src/Perfolizer/Perfolizer.Tests/Perfolizer.Tests.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -7,16 +7,27 @@
</PropertyGroup>

<ItemGroup>
<PackageReference Include="JetBrains.Annotations" Version="2019.1.3" PrivateAssets="All" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.2.0" />
<PackageReference Include="xunit" Version="2.4.0" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.4.0" />
<PackageReference Include="coverlet.collector" Version="1.0.1" />
<PackageReference Include="JetBrains.Annotations" Version="2024.2.0" PrivateAssets="All" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.11.0" />
<PackageReference Include="xunit" Version="2.9.0" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.8.2">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Include="coverlet.collector" Version="6.0.2">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Include="Verify.Xunit" Version="26.2.0" />
</ItemGroup>

<ItemGroup>
<ProjectReference Include="..\Perfolizer\Perfolizer.csproj" />
</ItemGroup>

<ItemGroup>
<Folder Include="Verify\" />
</ItemGroup>


</Project>
34 changes: 34 additions & 0 deletions src/Perfolizer/Perfolizer.Tests/Phd/JsonHelper.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
using System.Text.Json;
using System.Text.Json.Serialization;
using System.Text.Json.Serialization.Metadata;
using Perfolizer.Phd;
using Perfolizer.Phd.Base;

namespace Perfolizer.Tests.Phd;

public class PolymorphicTypeResolver(PhdSchema schema) : DefaultJsonTypeInfoResolver
{
public override JsonTypeInfo GetTypeInfo(Type type, JsonSerializerOptions options)
{
var jsonTypeInfo = base.GetTypeInfo(type, options);

foreach (var implementation in schema.Implementations)
{
if (jsonTypeInfo.Type == implementation.Base)
{
jsonTypeInfo.PolymorphismOptions = new JsonPolymorphismOptions
{
TypeDiscriminatorPropertyName = "$type",
IgnoreUnrecognizedTypeDiscriminators = true,
UnknownDerivedTypeHandling = JsonUnknownDerivedTypeHandling.FailSerialization,
DerivedTypes =
{
new JsonDerivedType(implementation.Derived, schema.Name)
}
};
}
}

return jsonTypeInfo;
}
}
21 changes: 21 additions & 0 deletions src/Perfolizer/Perfolizer.Tests/Phd/ModuleInit.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
using System.Runtime.CompilerServices;

namespace Perfolizer.Tests.Phd;

public static class ModuleInit
{
[ModuleInitializer]
public static void Init()
{
VerifierSettings.UseStrictJson();
VerifierSettings.AutoVerify();
VerifierSettings.DontScrubGuids();
VerifierSettings.DontScrubDateTimes();
VerifierSettings.DontSortDictionaries();
}

[ModuleInitializer]
public static void InitDerivePathInfo() =>
DerivePathInfo(
(_, _, type, method) => new(AttributeReader.GetProjectDirectory(), typeName: type.Name, method.Name));
}
80 changes: 80 additions & 0 deletions src/Perfolizer/Perfolizer.Tests/Phd/PhdBdnTests.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
using JetBrains.Annotations;
using Perfolizer.Horology;
using Perfolizer.Phd;

namespace Perfolizer.Tests.Phd;

// public class PhdBdnTests : PhdTestsBase
// {
// [Fact]
// public Task PhdBdn()
// {
// PerfMetric Metric(double value, int iterationIndex) => new()
// { Value = value, Unit = TimeUnit.Nanosecond, IterationIndex = iterationIndex, InvocationCount = 2 };
//
// var root = new PhdAttributes
// {
// Info = new BdnInfo { Title = "BenchmarkDotNet.Samples.IntroExportJson-20240309-013216" },
// Host = new BdnHost
// {
// BenchmarkDotNetCaption = "BenchmarkDotNet",
// BenchmarkDotNetVersion = "0.13.13-develop (2024-03-09)",
// OsVersion = "macOS Sonoma 14.2.1 (23C71) [Darwin 23.2.0]",
// ProcessorName = "Apple M1 Max",
// PhysicalProcessorCount = "1",
// PhysicalCoreCount = "10",
// LogicalCoreCount = "10",
// RuntimeVersion = ".NET 8.0.0 (8.0.23.53103)",
// Architecture = "Arm64",
// HasAttachedDebugger = false,
// HasRyuJit = true,
// Configuration = "RELEASE",
// DotNetSdkVersion = "8.0.100",
// ChronometerFrequency = 1000000000,
// HardwareTimerKind = "Unknown"
// }
// }.ToPerfEntry().Add(
// new PhdAttributes
// {
// Descriptor = new BdnDescriptor
// {
// DisplayInfo = "BenchmarkDotNet.Samples.IntroExportJson-20240309-013216",
// Namespace = "BenchmarkDotNet.Samples",
// Type = "IntroExportJson",
// Method = "ExportJson",
// MethodTitle = "ExportJson",
// Parameters = "Foo=1"
// }
// }.ToPerfEntry()
// .Add(new PhdAttributes
// {
// Lifecycle = new BdnLifecycle
// {
// IterationMode = "Pilot",
// IterationStage = "Overhead",
// LaunchIndex = 0
// }
// }.ToPerfEntry()
// .Add(Metric(12, 1))
// .Add(Metric(13, 2))
// .Add(Metric(14, 3)))
// .Add(new PhdAttributes
// {
// Lifecycle = new BdnLifecycle
// {
// IterationMode = "Result",
// IterationStage = "Workload",
// LaunchIndex = 1
// }
// }.ToPerfEntry()
// .Add(Metric(15, 1))
// .Add(Metric(16, 2))
// .Add(Metric(17, 3)))
// );
//
//
// return VerifyPhd(root, schema);
// }
//
//
// }
11 changes: 11 additions & 0 deletions src/Perfolizer/Perfolizer.Tests/Phd/PhdEmptyTests.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
using Perfolizer.Phd;
using Perfolizer.Phd.Base;
using Perfolizer.Phd.Dto;

namespace Perfolizer.Tests.Phd;

public class PhdEmptyTests : PhdTestsBase
{
[Fact]
public Task PhdEmpty() => VerifyPhd(new PhdEntry(new PhdAttributes()), new PhdSchema(""));
}
109 changes: 109 additions & 0 deletions src/Perfolizer/Perfolizer.Tests/Phd/PhdPerforatorTests.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,109 @@
using JetBrains.Annotations;
using Perfolizer.Horology;
using Perfolizer.Mathematics.Distributions.ContinuousDistributions;
using Perfolizer.Metrology;
using Perfolizer.Phd;
using Perfolizer.Phd.Base;
using Perfolizer.Phd.Dto;

namespace Perfolizer.Tests.Phd;

public class PhdPerforatorTests : PhdTestsBase
{
[Fact]
public Task PhdPerforator()
{
var runId = Guid.Parse("11214D6B-4E25-44A4-8032-D4290C9F5617");
var random = new NormalDistribution(10).Random(1729);
double NextValue() => Math.Round(random.Next(), 3);
int minute = 0;

PhdEntry CreateEntry(string benchmarkId) => new PhdAttributes
{
Identity = new PerforatorIdentity
{
RunId = runId,
Timestamp = DateTimeOffset.Parse($"2021-01-01T00:0{minute++}:00Z").ToUnixTimeMilliseconds()
},
Source = new PerforatorSource
{
BuildId = 123456,
Branch = "main",
ConfigurationId = "Configuration1",
},
Host = new PerforatorHost
{
Os = "Linux-x64",
ImageId = "Environment1",
},
Descriptor = new PerforatorDescriptor
{
BenchmarkId = benchmarkId,
BenchmarkVersion = 1,
},
}.ToPerfEntry()
.Add(new PhdMetric { Id = "stage1", Value = NextValue(), Unit = TimeUnit.Millisecond })
.Add(new PhdMetric { Id = "stage2", Version = 2, Value = NextValue(), Unit = TimeUnit.Millisecond })
.Add(new PhdMetric { Id = "totalTime", Value = NextValue(), Unit = TimeUnit.Millisecond })
.Add(new PhdMetric { Id = "Footprint", Value = 20, Unit = SizeUnit.MB })
.Add(new PhdMetric { Id = "Gc.CollectCount", Value = 3 });

var root = new PhdAttributes
{
Info = new PerforatorInfo { Title = "Perforator Measurements" }
}.ToPerfEntry()
.Add(CreateEntry("benchmark1"))
.Add(CreateEntry("benchmark1"))
.Add(CreateEntry("benchmark2"))
.Add(CreateEntry("benchmark2"));

var schema = new PhdSchema("perforator")
.Add<PerforatorInfo>()
.Add<PerforatorHost>()
.Add<PerforatorExecution>()
.Add<PerforatorDescriptor>()
.Add<PerforatorIdentity>()
.Add<PerforatorSource>();

return VerifyPhd(root, schema);
}

private class PerforatorInfo : PhdInfo
{
public string Title { get; set; } = "";
}

[PublicAPI]
private class PerforatorHost : PhdHost
{
public string Os { get; set; } = "";

Check warning on line 79 in src/Perfolizer/Perfolizer.Tests/Phd/PhdPerforatorTests.cs

View workflow job for this annotation

GitHub Actions / build-nix (macOS-latest)

'PhdPerforatorTests.PerforatorHost.Os' hides inherited member 'PhdHost.Os'. Use the new keyword if hiding was intended.

Check warning on line 79 in src/Perfolizer/Perfolizer.Tests/Phd/PhdPerforatorTests.cs

View workflow job for this annotation

GitHub Actions / build-nix (ubuntu-latest)

'PhdPerforatorTests.PerforatorHost.Os' hides inherited member 'PhdHost.Os'. Use the new keyword if hiding was intended.

Check warning on line 79 in src/Perfolizer/Perfolizer.Tests/Phd/PhdPerforatorTests.cs

View workflow job for this annotation

GitHub Actions / build-windows

'PhdPerforatorTests.PerforatorHost.Os' hides inherited member 'PhdHost.Os'. Use the new keyword if hiding was intended.

Check warning on line 79 in src/Perfolizer/Perfolizer.Tests/Phd/PhdPerforatorTests.cs

View workflow job for this annotation

GitHub Actions / build-nix (macOS-latest)

'PhdPerforatorTests.PerforatorHost.Os' hides inherited member 'PhdHost.Os'. Use the new keyword if hiding was intended.

Check warning on line 79 in src/Perfolizer/Perfolizer.Tests/Phd/PhdPerforatorTests.cs

View workflow job for this annotation

GitHub Actions / build-nix (ubuntu-latest)

'PhdPerforatorTests.PerforatorHost.Os' hides inherited member 'PhdHost.Os'. Use the new keyword if hiding was intended.

Check warning on line 79 in src/Perfolizer/Perfolizer.Tests/Phd/PhdPerforatorTests.cs

View workflow job for this annotation

GitHub Actions / build-windows

'PhdPerforatorTests.PerforatorHost.Os' hides inherited member 'PhdHost.Os'. Use the new keyword if hiding was intended.
public string ImageId { get; set; } = "";
}

[PublicAPI]
private class PerforatorExecution : PhdExecution
{
}

[PublicAPI]
private class PerforatorDescriptor : PhdDescriptor
{
public string BenchmarkId { get; set; } = "";
public int BenchmarkVersion { get; set; }
}

[PublicAPI]
private class PerforatorIdentity : PhdIdentity
{
public Guid RunId { get; set; }
public long Timestamp { get; set; }
}

[PublicAPI]
private class PerforatorSource : PhdSource
{
public int BuildId { get; set; }
public string ConfigurationId { get; set; } = "";
public string Branch { get; set; } = "";
}
}
33 changes: 33 additions & 0 deletions src/Perfolizer/Perfolizer.Tests/Phd/PhdTestsBase.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
using System.Text.Json;
using System.Text.Json.Serialization;
using Perfolizer.Json;
using Perfolizer.Phd;
using Perfolizer.Phd.Base;

namespace Perfolizer.Tests.Phd;

public class PhdTestsBase
{
protected static Task VerifyPhd(PhdEntry entry, PhdSchema schema)
{
// JsonSerializerOptions jsonOptions = new()
// {
// DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
// PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
// TypeInfoResolver = new PolymorphicTypeResolver(schema)
// };
// string json = JsonSerializer.Serialize(entry.Serialize(), jsonOptions);
string json = LightJsonSerializer.Serialize(entry.ToRaw());
return VerifyJson(json, InitSettings());
}

private static VerifySettings InitSettings()
{
var settings = new VerifySettings();
settings.UseDirectory("VerifiedFiles");
settings.UseTypeName("Phd");
settings.DisableDiff();
;
return settings;
}
}
Loading

0 comments on commit 5692c7e

Please sign in to comment.