From 5edf433d9af0d195ea6959ec84d3e38d120aa746 Mon Sep 17 00:00:00 2001 From: Craig Edmunds Date: Fri, 6 Dec 2024 10:01:29 +0000 Subject: [PATCH] CDMS-179 patches CDMS change to BTMS (#3) --- .../AggregationTestFixture.cs | 61 --------- .../Btms.Analytics.Tests.csproj | 1 + .../Fixtures/BasicSampleDataTestCollection.cs | 11 ++ .../Fixtures/BasicSampleDataTestFixture.cs | 66 +++++++++ .../MultiItemDataTestCollection.cs} | 6 +- .../Fixtures/MultiItemDataTestFixture.cs | 49 +++++++ .../Helpers/MultiSeriesDatasetAssertions.cs | 26 ++++ .../Helpers/SingleSeriesDatasetAssertions.cs | 16 +++ .../Helpers/TestAssertionExtensions.cs | 13 ++ .../Helpers/TestContextHelper.cs | 3 + ... ImportNotificationsByArrivalDateTests.cs} | 10 +- .../ImportNotificationsByCommoditiesTests.cs | 40 ++++++ ... ImportNotificationsByCreatedDateTests.cs} | 15 +- ...cs => ImportNotificationsByStatusTests.cs} | 14 +- ...ests.cs => MovementsByArrivalDateTests.cs} | 13 +- ...ests.cs => MovementsByCreatedDateTests.cs} | 14 +- Btms.Analytics.Tests/MovementsByItemsTests.cs | 40 ++++++ ...ntsStatus.cs => MovementsByStatusTests.cs} | 14 +- ...MovementsByUniqueDocumentReferenceTests.cs | 41 ++++++ ...ementsDocumentReferencesByMovementTests.cs | 27 ++++ Btms.Analytics/ByDateTimeResult.cs | 19 --- .../Extensions/AnalyticsExtensions.cs | 68 +++++++++- Btms.Analytics/Extensions/AnalyticsHelpers.cs | 22 ++- .../IImportNotificationsAggregationService.cs | 7 +- .../IMovementsAggregationService.cs | 9 +- Btms.Analytics/ImportNotificationMetrics.cs | 2 +- .../ImportNotificationsAggregationService.cs | 75 ++++++++-- Btms.Analytics/MovementsAggregationService.cs | 128 ++++++++++++++++-- Btms.Analytics/Results.cs | 31 +++++ Btms.Backend/Endpoints/AnalyticsEndpoints.cs | 58 +++++--- TestDataGenerator/ClearanceRequestBuilder.cs | 24 +++- .../Helpers/BuilderExtensions.cs | 2 +- TestDataGenerator/Program.cs | 3 +- TestDataGenerator/README.md | 2 +- .../Scenarios/CRNoMatchScenarioGenerator.cs | 3 +- 35 files changed, 748 insertions(+), 185 deletions(-) delete mode 100644 Btms.Analytics.Tests/AggregationTestFixture.cs create mode 100644 Btms.Analytics.Tests/Fixtures/BasicSampleDataTestCollection.cs create mode 100644 Btms.Analytics.Tests/Fixtures/BasicSampleDataTestFixture.cs rename Btms.Analytics.Tests/{AggregationTestCollection.cs => Fixtures/MultiItemDataTestCollection.cs} (51%) create mode 100644 Btms.Analytics.Tests/Fixtures/MultiItemDataTestFixture.cs create mode 100644 Btms.Analytics.Tests/Helpers/MultiSeriesDatasetAssertions.cs create mode 100644 Btms.Analytics.Tests/Helpers/SingleSeriesDatasetAssertions.cs create mode 100644 Btms.Analytics.Tests/Helpers/TestAssertionExtensions.cs rename Btms.Analytics.Tests/{GetImportNotificationsByArrivalDateTests.cs => ImportNotificationsByArrivalDateTests.cs} (76%) create mode 100644 Btms.Analytics.Tests/ImportNotificationsByCommoditiesTests.cs rename Btms.Analytics.Tests/{GetImportNotificationsByCreatedDateTests.cs => ImportNotificationsByCreatedDateTests.cs} (82%) rename Btms.Analytics.Tests/{GetImportNotificationsStatus.cs => ImportNotificationsByStatusTests.cs} (76%) rename Btms.Analytics.Tests/{GetMovementsByArrivalDateTests.cs => MovementsByArrivalDateTests.cs} (77%) rename Btms.Analytics.Tests/{GetMovementsByCreatedDateTests.cs => MovementsByCreatedDateTests.cs} (82%) create mode 100644 Btms.Analytics.Tests/MovementsByItemsTests.cs rename Btms.Analytics.Tests/{GetMovementsStatus.cs => MovementsByStatusTests.cs} (78%) create mode 100644 Btms.Analytics.Tests/MovementsByUniqueDocumentReferenceTests.cs create mode 100644 Btms.Analytics.Tests/MovementsDocumentReferencesByMovementTests.cs delete mode 100644 Btms.Analytics/ByDateTimeResult.cs create mode 100644 Btms.Analytics/Results.cs diff --git a/Btms.Analytics.Tests/AggregationTestFixture.cs b/Btms.Analytics.Tests/AggregationTestFixture.cs deleted file mode 100644 index 4110f05..0000000 --- a/Btms.Analytics.Tests/AggregationTestFixture.cs +++ /dev/null @@ -1,61 +0,0 @@ -using Btms.Analytics.Tests.Helpers; -using Btms.Analytics; -using Btms.Backend.Data; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Hosting; -using TestDataGenerator.Scenarios; - -namespace Btms.Analytics.Tests; - -#pragma warning disable S3881 -public class AggregationTestFixture : IDisposable -#pragma warning restore S3881 -{ - public IHost App; - public IImportNotificationsAggregationService ImportNotificationsAggregationService; - public IMovementsAggregationService MovementsAggregationService; - - public IMongoDbContext MongoDbContext; - public AggregationTestFixture() - { - var builder = TestContextHelper.CreateBuilder(); - - App = builder.Build(); - var rootScope = App.Services.CreateScope(); - - MongoDbContext = rootScope.ServiceProvider.GetRequiredService(); - ImportNotificationsAggregationService = rootScope.ServiceProvider.GetRequiredService(); - MovementsAggregationService = rootScope.ServiceProvider.GetRequiredService(); - - MongoDbContext.ResetCollections().GetAwaiter().GetResult(); - - // Ensure we have some data scenarios around 24/48 hour tests - App.PushToConsumers(App.CreateScenarioConfig(10, 3, arrivalDateRange: 0)) - .GetAwaiter().GetResult(); - - App.PushToConsumers(App.CreateScenarioConfig(10, 3, arrivalDateRange: 2)) - .GetAwaiter().GetResult(); - - App.PushToConsumers(App.CreateScenarioConfig(10, 3, arrivalDateRange: 0)) - .GetAwaiter().GetResult(); - - // Create some more variable data over the rest of time - App.PushToConsumers(App.CreateScenarioConfig(10, 7, arrivalDateRange: 10)) - .GetAwaiter().GetResult(); - - App.PushToConsumers(App.CreateScenarioConfig(5, 3, arrivalDateRange: 10)) - .GetAwaiter().GetResult(); - - App.PushToConsumers(App.CreateScenarioConfig(1, 3, arrivalDateRange: 10)) - .GetAwaiter().GetResult(); - - App.PushToConsumers(App.CreateScenarioConfig(1, 3, arrivalDateRange: 10)) - .GetAwaiter().GetResult(); - - } - - public void Dispose() - { - // ... clean up test data from the database ... - } -} \ No newline at end of file diff --git a/Btms.Analytics.Tests/Btms.Analytics.Tests.csproj b/Btms.Analytics.Tests/Btms.Analytics.Tests.csproj index ce5ca5e..a246796 100644 --- a/Btms.Analytics.Tests/Btms.Analytics.Tests.csproj +++ b/Btms.Analytics.Tests/Btms.Analytics.Tests.csproj @@ -7,6 +7,7 @@ false true + Btms.Analytics.Tests true diff --git a/Btms.Analytics.Tests/Fixtures/BasicSampleDataTestCollection.cs b/Btms.Analytics.Tests/Fixtures/BasicSampleDataTestCollection.cs new file mode 100644 index 0000000..7d4e2f7 --- /dev/null +++ b/Btms.Analytics.Tests/Fixtures/BasicSampleDataTestCollection.cs @@ -0,0 +1,11 @@ +using Xunit; + +namespace Btms.Analytics.Tests.Fixtures; + +[CollectionDefinition(nameof(BasicSampleDataTestCollection))] +public class BasicSampleDataTestCollection : ICollectionFixture +{ + // This class has no code, and is never created. Its purpose is simply + // to be the place to apply [CollectionDefinition] and all the + // ICollectionFixture<> interfaces. +} \ No newline at end of file diff --git a/Btms.Analytics.Tests/Fixtures/BasicSampleDataTestFixture.cs b/Btms.Analytics.Tests/Fixtures/BasicSampleDataTestFixture.cs new file mode 100644 index 0000000..5ee8684 --- /dev/null +++ b/Btms.Analytics.Tests/Fixtures/BasicSampleDataTestFixture.cs @@ -0,0 +1,66 @@ +using Btms.Analytics.Tests.Helpers; +using Btms.Backend.Data; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using TestDataGenerator.Scenarios; + +namespace Btms.Analytics.Tests.Fixtures; + +#pragma warning disable S3881 +public class BasicSampleDataTestFixture : IDisposable +#pragma warning restore S3881 +{ + public IHost App; + public IImportNotificationsAggregationService ImportNotificationsAggregationService; + public IMovementsAggregationService MovementsAggregationService; + + public IMongoDbContext MongoDbContext; + public BasicSampleDataTestFixture() + { + var builder = TestContextHelper.CreateBuilder(); + + App = builder.Build(); + var rootScope = App.Services.CreateScope(); + + MongoDbContext = rootScope.ServiceProvider.GetRequiredService(); + ImportNotificationsAggregationService = rootScope.ServiceProvider.GetRequiredService(); + MovementsAggregationService = rootScope.ServiceProvider.GetRequiredService(); + + // Would like to pick this up from env/config/DB state + var insertToMongo = true; + + if (insertToMongo) + { + MongoDbContext.ResetCollections().GetAwaiter().GetResult(); + + // Ensure we have some data scenarios around 24/48 hour tests + App.PushToConsumers(App.CreateScenarioConfig(10, 3, arrivalDateRange: 0)) + .GetAwaiter().GetResult(); + + App.PushToConsumers(App.CreateScenarioConfig(10, 3, arrivalDateRange: 2)) + .GetAwaiter().GetResult(); + + App.PushToConsumers(App.CreateScenarioConfig(10, 3, arrivalDateRange: 0)) + .GetAwaiter().GetResult(); + + // Create some more variable data over the rest of time + App.PushToConsumers( + App.CreateScenarioConfig(10, 7, arrivalDateRange: 10)) + .GetAwaiter().GetResult(); + + App.PushToConsumers(App.CreateScenarioConfig(5, 3, arrivalDateRange: 10)) + .GetAwaiter().GetResult(); + + App.PushToConsumers(App.CreateScenarioConfig(1, 3, arrivalDateRange: 10)) + .GetAwaiter().GetResult(); + + App.PushToConsumers(App.CreateScenarioConfig(1, 3, arrivalDateRange: 10)) + .GetAwaiter().GetResult(); + } + } + + public void Dispose() + { + // ... clean up test data from the database ... + } +} \ No newline at end of file diff --git a/Btms.Analytics.Tests/AggregationTestCollection.cs b/Btms.Analytics.Tests/Fixtures/MultiItemDataTestCollection.cs similarity index 51% rename from Btms.Analytics.Tests/AggregationTestCollection.cs rename to Btms.Analytics.Tests/Fixtures/MultiItemDataTestCollection.cs index 0bb7712..2c37b83 100644 --- a/Btms.Analytics.Tests/AggregationTestCollection.cs +++ b/Btms.Analytics.Tests/Fixtures/MultiItemDataTestCollection.cs @@ -1,9 +1,9 @@ using Xunit; -namespace Btms.Analytics.Tests; +namespace Btms.Analytics.Tests.Fixtures; -[CollectionDefinition("Aggregation Test collection")] -public class AggregationTestCollection : ICollectionFixture +[CollectionDefinition(nameof(MultiItemDataTestCollection))] +public class MultiItemDataTestCollection : ICollectionFixture { // This class has no code, and is never created. Its purpose is simply // to be the place to apply [CollectionDefinition] and all the diff --git a/Btms.Analytics.Tests/Fixtures/MultiItemDataTestFixture.cs b/Btms.Analytics.Tests/Fixtures/MultiItemDataTestFixture.cs new file mode 100644 index 0000000..c322101 --- /dev/null +++ b/Btms.Analytics.Tests/Fixtures/MultiItemDataTestFixture.cs @@ -0,0 +1,49 @@ +using Btms.Analytics.Tests.Helpers; +using Btms.Backend.Data; +using Microsoft.Extensions.DependencyInjection; +using TestDataGenerator.Scenarios; + +namespace Btms.Analytics.Tests.Fixtures; + +#pragma warning disable S3881 +public class MultiItemDataTestFixture : IDisposable +#pragma warning restore S3881 +{ + public readonly IImportNotificationsAggregationService ImportNotificationsAggregationService; + public readonly IMovementsAggregationService MovementsAggregationService; + + public IMongoDbContext MongoDbContext; + public MultiItemDataTestFixture() + { + var builder = TestContextHelper.CreateBuilder(); + + var app = builder.Build(); + var rootScope = app.Services.CreateScope(); + + MongoDbContext = rootScope.ServiceProvider.GetRequiredService(); + ImportNotificationsAggregationService = rootScope.ServiceProvider.GetRequiredService(); + MovementsAggregationService = rootScope.ServiceProvider.GetRequiredService(); + + // Would like to pick this up from env/config/DB state + var insertToMongo = true; + + if (insertToMongo) + { + MongoDbContext.ResetCollections().GetAwaiter().GetResult(); + + app.PushToConsumers(app.CreateScenarioConfig(10, 3, arrivalDateRange: 0)) + .GetAwaiter().GetResult(); + + app.PushToConsumers(app.CreateScenarioConfig(10, 3, arrivalDateRange: 0)) + .GetAwaiter().GetResult(); + + app.PushToConsumers(app.CreateScenarioConfig(10, 3, arrivalDateRange: 0)) + .GetAwaiter().GetResult(); + } + } + + public void Dispose() + { + // ... clean up test data from the database ... + } +} \ No newline at end of file diff --git a/Btms.Analytics.Tests/Helpers/MultiSeriesDatasetAssertions.cs b/Btms.Analytics.Tests/Helpers/MultiSeriesDatasetAssertions.cs new file mode 100644 index 0000000..d2c8140 --- /dev/null +++ b/Btms.Analytics.Tests/Helpers/MultiSeriesDatasetAssertions.cs @@ -0,0 +1,26 @@ +using Btms.Common.Extensions; +using FluentAssertions; +using FluentAssertions.Collections; + +namespace Btms.Analytics.Tests.Helpers; + +public class MultiSeriesDatasetAssertions(List? test) + : GenericCollectionAssertions(test) +{ + [CustomAssertion] + public void BeSameLength(string because = "", params object[] becauseArgs) + { + test!.Select(r => r.Results.Count) + .Distinct() + .Count() + .Should() + .Be(1); + } + + [CustomAssertion] + public void HaveResults(string because = "", params object[] becauseArgs) + { + test!.Sum(d => d.Results.Sum(r => r.Value)) + .Should().BeGreaterThan(0); + } +} \ No newline at end of file diff --git a/Btms.Analytics.Tests/Helpers/SingleSeriesDatasetAssertions.cs b/Btms.Analytics.Tests/Helpers/SingleSeriesDatasetAssertions.cs new file mode 100644 index 0000000..19cb006 --- /dev/null +++ b/Btms.Analytics.Tests/Helpers/SingleSeriesDatasetAssertions.cs @@ -0,0 +1,16 @@ +using Btms.Common.Extensions; +using FluentAssertions; +using FluentAssertions.Collections; + +namespace Btms.Analytics.Tests.Helpers; + +public class SingleSeriesDatasetAssertions(SingeSeriesDataset? test) +{ + [CustomAssertion] + public void HaveResults(string because = "", params object[] becauseArgs) + { + test!.Values + .Values.Sum() + .Should().BeGreaterThan(0); + } +} \ No newline at end of file diff --git a/Btms.Analytics.Tests/Helpers/TestAssertionExtensions.cs b/Btms.Analytics.Tests/Helpers/TestAssertionExtensions.cs new file mode 100644 index 0000000..2fbd224 --- /dev/null +++ b/Btms.Analytics.Tests/Helpers/TestAssertionExtensions.cs @@ -0,0 +1,13 @@ +namespace Btms.Analytics.Tests.Helpers; + +public static class TestAssertionExtensions +{ + public static MultiSeriesDatasetAssertions Should(this List? instance) + { + return new MultiSeriesDatasetAssertions(instance); + } + public static SingleSeriesDatasetAssertions Should(this SingeSeriesDataset? instance) + { + return new SingleSeriesDatasetAssertions(instance); + } +} \ No newline at end of file diff --git a/Btms.Analytics.Tests/Helpers/TestContextHelper.cs b/Btms.Analytics.Tests/Helpers/TestContextHelper.cs index 4a621e6..a9bd538 100644 --- a/Btms.Analytics.Tests/Helpers/TestContextHelper.cs +++ b/Btms.Analytics.Tests/Helpers/TestContextHelper.cs @@ -8,8 +8,11 @@ using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using TestDataGenerator.Helpers; +using Xunit; using Xunit.Abstractions; +[assembly: CollectionBehavior(DisableTestParallelization = true)] + namespace Btms.Analytics.Tests.Helpers; public static class TestContextHelper diff --git a/Btms.Analytics.Tests/GetImportNotificationsByArrivalDateTests.cs b/Btms.Analytics.Tests/ImportNotificationsByArrivalDateTests.cs similarity index 76% rename from Btms.Analytics.Tests/GetImportNotificationsByArrivalDateTests.cs rename to Btms.Analytics.Tests/ImportNotificationsByArrivalDateTests.cs index da82416..cfc4a53 100644 --- a/Btms.Analytics.Tests/GetImportNotificationsByArrivalDateTests.cs +++ b/Btms.Analytics.Tests/ImportNotificationsByArrivalDateTests.cs @@ -3,11 +3,13 @@ using Xunit; using Xunit.Abstractions; +using Btms.Analytics.Tests.Fixtures; + namespace Btms.Analytics.Tests; -[Collection("Aggregation Test collection")] -public class GetImportNotificationsByArrivalDateTests( - AggregationTestFixture aggregationTestFixture, +[Collection(nameof(BasicSampleDataTestCollection))] +public class ImportNotificationsByArrivalDateTests( + BasicSampleDataTestFixture basicSampleDataTestFixture, ITestOutputHelper testOutputHelper) { @@ -16,7 +18,7 @@ public async Task WhenCalledNextMonth_ReturnExpectedAggregation() { testOutputHelper.WriteLine("Querying for aggregated data"); - var result = (await aggregationTestFixture.ImportNotificationsAggregationService + var result = (await basicSampleDataTestFixture.ImportNotificationsAggregationService .ByArrival(DateTime.Today, DateTime.Today.MonthLater())) .ToList(); diff --git a/Btms.Analytics.Tests/ImportNotificationsByCommoditiesTests.cs b/Btms.Analytics.Tests/ImportNotificationsByCommoditiesTests.cs new file mode 100644 index 0000000..c371553 --- /dev/null +++ b/Btms.Analytics.Tests/ImportNotificationsByCommoditiesTests.cs @@ -0,0 +1,40 @@ +using Btms.Analytics.Extensions; +using Btms.Common.Extensions; +using FluentAssertions; +using Xunit; +using Xunit.Abstractions; +using Btms.Analytics.Tests.Helpers; +using Btms.Analytics.Tests.Fixtures; + +namespace Btms.Analytics.Tests; + +[Collection(nameof(MultiItemDataTestCollection))] +public class ImportNotificationsByCommoditiesTests( + MultiItemDataTestFixture multiItemDataTestFixture, + ITestOutputHelper testOutputHelper) +{ + + [Fact] + public async Task WhenCalledLastWeek_ReturnExpectedAggregation() + { + testOutputHelper.WriteLine("Querying for aggregated data"); + var result = (await multiItemDataTestFixture.ImportNotificationsAggregationService + .ByCommodityCount(DateTime.Today.WeekAgo(), DateTime.Today.Tomorrow())) + .ToList();; + + testOutputHelper.WriteLine("{0} aggregated items found", result.Count); + + result.Count().Should().Be(8); + result.Select(r => r.Name).Order().Should().Equal(AnalyticsHelpers.GetImportNotificationSegments().Order()); + + result.Should().AllSatisfy(r => + { + r.Dimension.Should().Be("ItemCount"); + r.Results.Count().Should().NotBe(0); + }); + + result.Should().HaveResults(); + + result.Should().BeSameLength(); + } +} \ No newline at end of file diff --git a/Btms.Analytics.Tests/GetImportNotificationsByCreatedDateTests.cs b/Btms.Analytics.Tests/ImportNotificationsByCreatedDateTests.cs similarity index 82% rename from Btms.Analytics.Tests/GetImportNotificationsByCreatedDateTests.cs rename to Btms.Analytics.Tests/ImportNotificationsByCreatedDateTests.cs index 2d1b9b3..d750853 100644 --- a/Btms.Analytics.Tests/GetImportNotificationsByCreatedDateTests.cs +++ b/Btms.Analytics.Tests/ImportNotificationsByCreatedDateTests.cs @@ -1,23 +1,22 @@ -using Btms.Analytics; using Btms.Common.Extensions; using Btms.Model.Extensions; using FluentAssertions; using Xunit; using Xunit.Abstractions; -[assembly: CollectionBehavior(DisableTestParallelization = true)] +using Btms.Analytics.Tests.Fixtures; namespace Btms.Analytics.Tests; -[Collection("Aggregation Test collection")] -public class GetImportNotificationsByCreatedDateTests( - AggregationTestFixture aggregationTestFixture, +[Collection(nameof(BasicSampleDataTestCollection))] +public class ImportNotificationsByCreatedDateTests( + BasicSampleDataTestFixture basicSampleDataTestFixture, ITestOutputHelper testOutputHelper) { [Fact] public async Task WhenCalledLast48Hours_ReturnExpectedAggregation() { - var result = (await aggregationTestFixture.ImportNotificationsAggregationService + var result = (await basicSampleDataTestFixture.ImportNotificationsAggregationService .ByCreated(DateTime.Now.NextHour().AddDays(-2), DateTime.Now.NextHour(),AggregationPeriod.Hour)) .ToList(); @@ -35,7 +34,7 @@ public async Task WhenCalledLast48Hours_ReturnExpectedAggregation() [Fact] public async Task WhenCalledLastMonth_ReturnExpectedAggregation() { - var result = (await aggregationTestFixture.ImportNotificationsAggregationService + var result = (await basicSampleDataTestFixture.ImportNotificationsAggregationService .ByCreated(DateTime.Today.MonthAgo(), DateTime.Today.Tomorrow())) .ToList(); @@ -62,7 +61,7 @@ public async Task WhenCalledWithTimePeriodYieldingNoResults_ReturnEmptyAggregati DateTime from = DateTime.MaxValue.AddDays(-1); DateTime to = DateTime.MaxValue; - var result = (await aggregationTestFixture.ImportNotificationsAggregationService + var result = (await basicSampleDataTestFixture.ImportNotificationsAggregationService .ByCreated(from, to, AggregationPeriod.Hour)) .ToList(); diff --git a/Btms.Analytics.Tests/GetImportNotificationsStatus.cs b/Btms.Analytics.Tests/ImportNotificationsByStatusTests.cs similarity index 76% rename from Btms.Analytics.Tests/GetImportNotificationsStatus.cs rename to Btms.Analytics.Tests/ImportNotificationsByStatusTests.cs index c9109af..8ad2745 100644 --- a/Btms.Analytics.Tests/GetImportNotificationsStatus.cs +++ b/Btms.Analytics.Tests/ImportNotificationsByStatusTests.cs @@ -3,11 +3,13 @@ using Xunit; using Xunit.Abstractions; +using Btms.Analytics.Tests.Fixtures; + namespace Btms.Analytics.Tests; -[Collection("Aggregation Test collection")] -public class GetImportNotificationsStatus( - AggregationTestFixture aggregationTestFixture, +[Collection(nameof(BasicSampleDataTestCollection))] +public class ImportNotificationsByStatusTests( + BasicSampleDataTestFixture basicSampleDataTestFixture, ITestOutputHelper testOutputHelper) { @@ -15,7 +17,7 @@ public class GetImportNotificationsStatus( public async Task WhenCalledLastWeek_ReturnExpectedAggregation() { testOutputHelper.WriteLine("Querying for aggregated data"); - var result = (await aggregationTestFixture.ImportNotificationsAggregationService + var result = (await basicSampleDataTestFixture.ImportNotificationsAggregationService .ByStatus(DateTime.Today.WeekAgo(), DateTime.Today.Tomorrow())); testOutputHelper.WriteLine("{0} aggregated items found", result.Values.Count); @@ -27,7 +29,7 @@ public async Task WhenCalledLastWeek_ReturnExpectedAggregation() public async Task WhenCalledLast48Hours_ReturnExpectedAggregation() { testOutputHelper.WriteLine("Querying for aggregated data"); - var result = (await aggregationTestFixture.ImportNotificationsAggregationService + var result = (await basicSampleDataTestFixture.ImportNotificationsAggregationService .ByStatus(DateTime.Now.NextHour().AddDays(-2), DateTime.Now.NextHour())); testOutputHelper.WriteLine($"{result.Values.Count} aggregated items found"); @@ -42,7 +44,7 @@ public async Task WhenCalledLast48Hours_ReturnExpectedAggregation() public async Task WhenCalledWithTimePeriodYieldingNoResults_ReturnEmptyAggregation() { testOutputHelper.WriteLine("Querying for aggregated data"); - var result = (await aggregationTestFixture.ImportNotificationsAggregationService + var result = (await basicSampleDataTestFixture.ImportNotificationsAggregationService .ByStatus(DateTime.MaxValue.AddDays(-1), DateTime.MaxValue)); testOutputHelper.WriteLine($"{result.Values.Count} aggregated items found"); diff --git a/Btms.Analytics.Tests/GetMovementsByArrivalDateTests.cs b/Btms.Analytics.Tests/MovementsByArrivalDateTests.cs similarity index 77% rename from Btms.Analytics.Tests/GetMovementsByArrivalDateTests.cs rename to Btms.Analytics.Tests/MovementsByArrivalDateTests.cs index 00e29db..3e36d79 100644 --- a/Btms.Analytics.Tests/GetMovementsByArrivalDateTests.cs +++ b/Btms.Analytics.Tests/MovementsByArrivalDateTests.cs @@ -1,22 +1,23 @@ -using Btms.Analytics; using Btms.Common.Extensions; using Btms.Model.Extensions; using FluentAssertions; using Xunit; using Xunit.Abstractions; +using Btms.Analytics.Tests.Fixtures; + namespace Btms.Analytics.Tests; -[Collection("Aggregation Test collection")] -public class GetMovementsByArrivalDateTests( - AggregationTestFixture aggregationTestFixture, +[Collection(nameof(BasicSampleDataTestCollection))] +public class MovementsByArrivalDateTests( + BasicSampleDataTestFixture basicSampleDataTestFixture, ITestOutputHelper testOutputHelper) { [Fact] public async Task WhenCalledNext72Hours_ReturnExpectedAggregation() { - var result = (await aggregationTestFixture.MovementsAggregationService + var result = (await basicSampleDataTestFixture.MovementsAggregationService .ByArrival(DateTime.Now.CurrentHour(), DateTime.Now.CurrentHour().AddDays(3), AggregationPeriod.Hour)) .ToList(); @@ -31,7 +32,7 @@ public async Task WhenCalledNext72Hours_ReturnExpectedAggregation() [Fact] public async Task WhenCalledNextMonth_ReturnExpectedAggregation() { - var result = (await aggregationTestFixture.MovementsAggregationService + var result = (await basicSampleDataTestFixture.MovementsAggregationService .ByArrival(DateTime.Today, DateTime.Today.MonthLater())) .ToList(); diff --git a/Btms.Analytics.Tests/GetMovementsByCreatedDateTests.cs b/Btms.Analytics.Tests/MovementsByCreatedDateTests.cs similarity index 82% rename from Btms.Analytics.Tests/GetMovementsByCreatedDateTests.cs rename to Btms.Analytics.Tests/MovementsByCreatedDateTests.cs index 2333ca0..e57094b 100644 --- a/Btms.Analytics.Tests/GetMovementsByCreatedDateTests.cs +++ b/Btms.Analytics.Tests/MovementsByCreatedDateTests.cs @@ -1,21 +1,21 @@ -using Btms.Analytics; using Btms.Common.Extensions; using Btms.Model.Extensions; using FluentAssertions; using Xunit; using Xunit.Abstractions; +using Btms.Analytics.Tests.Fixtures; namespace Btms.Analytics.Tests; -[Collection("Aggregation Test collection")] -public class GetMovementsByCreatedDateTests( - AggregationTestFixture aggregationTestFixture, +[Collection(nameof(BasicSampleDataTestCollection))] +public class MovementsByCreatedDateTests( + BasicSampleDataTestFixture basicSampleDataTestFixture, ITestOutputHelper testOutputHelper) { [Fact] public async Task WhenCalledLast48Hours_ReturnExpectedAggregation() { - var result = (await aggregationTestFixture.MovementsAggregationService + var result = (await basicSampleDataTestFixture.MovementsAggregationService .ByCreated(DateTime.Now.NextHour().AddDays(-2), DateTime.Now.NextHour(), AggregationPeriod.Hour)) .ToList(); @@ -36,7 +36,7 @@ public async Task WhenCalledWithTimePeriodYieldingNoResults_ReturnEmptyAggregati DateTime from = DateTime.MaxValue.AddDays(-1); DateTime to = DateTime.MaxValue; - var result = (await aggregationTestFixture.MovementsAggregationService + var result = (await basicSampleDataTestFixture.MovementsAggregationService .ByCreated(from, to, AggregationPeriod.Hour)) .ToList(); @@ -60,7 +60,7 @@ public async Task WhenCalledWithTimePeriodYieldingNoResults_ReturnEmptyAggregati [Fact] public async Task WhenCalledLastMonth_ReturnExpectedAggregation() { - var result = (await aggregationTestFixture.MovementsAggregationService + var result = (await basicSampleDataTestFixture.MovementsAggregationService .ByCreated(DateTime.Today.MonthAgo(), DateTime.Today.Tomorrow())) .ToList(); diff --git a/Btms.Analytics.Tests/MovementsByItemsTests.cs b/Btms.Analytics.Tests/MovementsByItemsTests.cs new file mode 100644 index 0000000..4575d59 --- /dev/null +++ b/Btms.Analytics.Tests/MovementsByItemsTests.cs @@ -0,0 +1,40 @@ +using Btms.Common.Extensions; +using FluentAssertions; +using Xunit; +using Xunit.Abstractions; + +using Btms.Analytics.Tests.Fixtures; +using Btms.Analytics.Tests.Helpers; + +namespace Btms.Analytics.Tests; + +[Collection(nameof(MultiItemDataTestCollection))] +public class MovementsByItemsTests( + MultiItemDataTestFixture multiItemDataTestFixture, + ITestOutputHelper testOutputHelper) +{ + + [Fact] + public async Task WhenCalledLastWeek_ReturnExpectedAggregation() + { + testOutputHelper.WriteLine("Querying for aggregated data"); + var result = (await multiItemDataTestFixture.MovementsAggregationService + .ByItemCount(DateTime.Today.WeekAgo(), DateTime.Today.Tomorrow())) + .ToList();; + + testOutputHelper.WriteLine("{0} aggregated items found", result.Count); + + result.Count().Should().Be(2); + result.Select(r => r.Name).Order().Should().Equal("Linked", "Not Linked"); + + result.Should().AllSatisfy(r => + { + r.Dimension.Should().Be("Item Count"); + r.Results.Count().Should().NotBe(0); + }); + + result.Should().HaveResults(); + + result.Should().BeSameLength(); + } +} \ No newline at end of file diff --git a/Btms.Analytics.Tests/GetMovementsStatus.cs b/Btms.Analytics.Tests/MovementsByStatusTests.cs similarity index 78% rename from Btms.Analytics.Tests/GetMovementsStatus.cs rename to Btms.Analytics.Tests/MovementsByStatusTests.cs index 3e7e817..ae9f02b 100644 --- a/Btms.Analytics.Tests/GetMovementsStatus.cs +++ b/Btms.Analytics.Tests/MovementsByStatusTests.cs @@ -3,11 +3,13 @@ using Xunit; using Xunit.Abstractions; +using Btms.Analytics.Tests.Fixtures; + namespace Btms.Analytics.Tests; -[Collection("Aggregation Test collection")] -public class GetMovementsStatus( - AggregationTestFixture aggregationTestFixture, +[Collection(nameof(BasicSampleDataTestCollection))] +public class MovementsByStatusTests( + BasicSampleDataTestFixture basicSampleDataTestFixture, ITestOutputHelper testOutputHelper) { @@ -15,7 +17,7 @@ public class GetMovementsStatus( public async Task WhenCalledLastWeek_ReturnExpectedAggregation() { testOutputHelper.WriteLine("Querying for aggregated data"); - var result = (await aggregationTestFixture.MovementsAggregationService + var result = (await basicSampleDataTestFixture.MovementsAggregationService .ByStatus(DateTime.Today.WeekAgo(), DateTime.Today.Tomorrow())); testOutputHelper.WriteLine("{0} aggregated items found", result.Values.Count); @@ -28,7 +30,7 @@ public async Task WhenCalledLastWeek_ReturnExpectedAggregation() public async Task WhenCalledLast48Hours_ReturnExpectedAggregation() { testOutputHelper.WriteLine("Querying for aggregated data"); - var result = (await aggregationTestFixture.MovementsAggregationService + var result = (await basicSampleDataTestFixture.MovementsAggregationService .ByStatus(DateTime.Now.NextHour().AddDays(-2), DateTime.Now.NextHour())); testOutputHelper.WriteLine($"{result.Values.Count} aggregated items found"); @@ -41,7 +43,7 @@ public async Task WhenCalledLast48Hours_ReturnExpectedAggregation() public async Task WhenCalledWithTimePeriodYieldingNoResults_ReturnEmptyAggregation() { testOutputHelper.WriteLine("Querying for aggregated data"); - var result = (await aggregationTestFixture.MovementsAggregationService + var result = (await basicSampleDataTestFixture.MovementsAggregationService .ByStatus(DateTime.MaxValue.AddDays(-1), DateTime.MaxValue)); testOutputHelper.WriteLine($"{result.Values.Count} aggregated items found"); diff --git a/Btms.Analytics.Tests/MovementsByUniqueDocumentReferenceTests.cs b/Btms.Analytics.Tests/MovementsByUniqueDocumentReferenceTests.cs new file mode 100644 index 0000000..1a4f6ca --- /dev/null +++ b/Btms.Analytics.Tests/MovementsByUniqueDocumentReferenceTests.cs @@ -0,0 +1,41 @@ +using Btms.Common.Extensions; +using FluentAssertions; +using Xunit; +using Xunit.Abstractions; + +using Btms.Analytics.Tests.Fixtures; +using Btms.Analytics.Tests.Helpers; + +namespace Btms.Analytics.Tests; + +[Collection(nameof(MultiItemDataTestCollection))] +public class MovementsByUniqueDocumentReferenceTests( + MultiItemDataTestFixture multiItemDataTestFixture, + ITestOutputHelper testOutputHelper) +{ + + [Fact] + public async Task WhenCalledLastWeek_ReturnExpectedAggregation() + { + testOutputHelper.WriteLine("Querying for aggregated data"); + var result = (await multiItemDataTestFixture.MovementsAggregationService + .ByUniqueDocumentReferenceCount(DateTime.Today.WeekAgo(), DateTime.Today.Tomorrow())) + .ToList();; + + testOutputHelper.WriteLine("{0} aggregated items found", result.Count); + + result.Count().Should().Be(2); + result.Select(r => r.Name).Order().Should().Equal("Linked", "Not Linked"); + + result.Should().AllSatisfy(d => + { + d.Dimension.Should().Be("Document Reference Count"); + d.Results.Count().Should().NotBe(0); + d.Results.Sum(r => r.Value).Should().BeGreaterThan(0); + }); + + result.Should().HaveResults(); + + result.Should().BeSameLength(); + } +} \ No newline at end of file diff --git a/Btms.Analytics.Tests/MovementsDocumentReferencesByMovementTests.cs b/Btms.Analytics.Tests/MovementsDocumentReferencesByMovementTests.cs new file mode 100644 index 0000000..c17239b --- /dev/null +++ b/Btms.Analytics.Tests/MovementsDocumentReferencesByMovementTests.cs @@ -0,0 +1,27 @@ +using Btms.Common.Extensions; +using FluentAssertions; +using Xunit; +using Xunit.Abstractions; + +using Btms.Analytics.Tests.Fixtures; +using Btms.Analytics.Tests.Helpers; + +namespace Btms.Analytics.Tests; + +[Collection(nameof(MultiItemDataTestCollection))] +public class MovementsDocumentReferencesByMovementTests( + MultiItemDataTestFixture multiItemDataTestFixture, + ITestOutputHelper testOutputHelper) +{ + [Fact] + public async Task WhenCalledLastWeek_ReturnExpectedAggregation() + { + testOutputHelper.WriteLine("Querying for aggregated data"); + var result = (await multiItemDataTestFixture.MovementsAggregationService + .UniqueDocumentReferenceByMovementCount(DateTime.Today.WeekAgo(), DateTime.Today.Tomorrow())); + + testOutputHelper.WriteLine("{0} aggregated items found", result.Values.Count); + + result.Should().HaveResults(); + } +} \ No newline at end of file diff --git a/Btms.Analytics/ByDateTimeResult.cs b/Btms.Analytics/ByDateTimeResult.cs deleted file mode 100644 index dde37d7..0000000 --- a/Btms.Analytics/ByDateTimeResult.cs +++ /dev/null @@ -1,19 +0,0 @@ -namespace Btms.Analytics; - -public class ByDateTimeResult -{ - public DateTime Period { get; set; } - public int Value { get; set; } -} - -public class PieChartDataset -{ - public IDictionary Values { get; set; } = new Dictionary(); -} - -public class Dataset(string name) -{ - public string Name { get; set; } = name; - public List Periods { get; set; } = []; - -} \ No newline at end of file diff --git a/Btms.Analytics/Extensions/AnalyticsExtensions.cs b/Btms.Analytics/Extensions/AnalyticsExtensions.cs index 4525b55..70b2e1c 100644 --- a/Btms.Analytics/Extensions/AnalyticsExtensions.cs +++ b/Btms.Analytics/Extensions/AnalyticsExtensions.cs @@ -1,12 +1,17 @@ +using System.Collections; +using System.Linq.Expressions; using Btms.Backend.Data; using Btms.Model.Data; +using Btms.Model.Extensions; using Btms.Model.Ipaffs; using Microsoft.AspNetCore.Components.Web; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Logging; using MongoDB.Bson; using MongoDB.Driver; +using MongoDB.Driver.Linq; namespace Btms.Analytics.Extensions; @@ -28,7 +33,7 @@ public static IServiceCollection AddAnalyticsServices(this IServiceCollection se return services; } - public static string MetricsKey(this Dataset ds) + public static string MetricsKey(this MultiSeriesDatetimeDataset ds) { return ds.Name.Replace(" ", "-").ToLower(); } @@ -64,10 +69,10 @@ public static Dictionary GetNamedSetAsDict(this Dictionary records, DateTime[] dateRange, string title) + public static MultiSeriesDatetimeDataset AsDataset(this Dictionary records, DateTime[] dateRange, string title) { var dates = records.GetNamedSetAsDict(title); - return new Dataset(title) + return new MultiSeriesDatetimeDataset(title) { Periods = dateRange .Select(resultDate => @@ -80,11 +85,64 @@ public static Dataset AsDataset(this Dictionary records, D }; } - public static Dataset[] AsOrderedArray(this IEnumerable en) + public static T[] AsOrderedArray(this IEnumerable en, Func keySelector) { return en - .OrderBy(d => d.Name) + .OrderBy(keySelector) .ToArray(); } + internal static IEnumerable> Execute(this IQueryable> source, ILogger logger) + { + + try + { + var aggregatedData = source.ToList(); + return aggregatedData; + } + catch(Exception ex) + { + logger.LogError(ex, "Error querying Mongo : {message}", ex.Message); + throw new AnalyticsException("Error querying Mongo", ex); + } + finally + { + logger.LogExecutedMongoString(source); + } + } + + internal static IEnumerable Execute( + this IQueryable source, ILogger logger) + { + + try + { + var aggregatedData = source.ToList(); + return aggregatedData; + } + catch(Exception ex) + { + logger.LogError(ex, "Error querying Mongo : {message}", ex.Message); + throw new AnalyticsException("Error querying Mongo", ex); + } + finally + { + logger.LogExecutedMongoString(source); + } + } + + /// + /// Gets the executed query details to allow issues to be reproduced + /// I'm sure this could be made cleaner but should do for now. + /// + /// Where to write the log + /// A mongo query that has already executed + private static void LogExecutedMongoString(this ILogger logger, IQueryable source) + { + var stages = ((IMongoQueryProvider)source.Provider).LoggedStages; + + var query = "[" + String.Join(",", stages.Select(s => s.ToString()).ToArray()) +"]"; + + logger.LogInformation(query); + } } \ No newline at end of file diff --git a/Btms.Analytics/Extensions/AnalyticsHelpers.cs b/Btms.Analytics/Extensions/AnalyticsHelpers.cs index 352874b..62ab413 100644 --- a/Btms.Analytics/Extensions/AnalyticsHelpers.cs +++ b/Btms.Analytics/Extensions/AnalyticsHelpers.cs @@ -1,19 +1,22 @@ using Btms.Common.Extensions; using Btms.Model.Extensions; using System.Collections.Generic; +using Btms.Model; using MongoDB.Bson; namespace Btms.Analytics.Extensions; +public class AnalyticsException(string message, Exception inner) : Exception(message, inner); + public static class AnalyticsMetricNames { public const string MeterName = "Btms.Backend.Analytics"; - public const string MetricPrefix = "btms.service.analytics"; + public const string MetricPrefix = "Btms.service.analytics"; public static class CommonTags { - public const string Service = "btms.service.anayltics"; - public const string ExceptionType = "btms.exception_type"; - public const string MessageType = "btms.message_type"; + public const string Service = "Btms.service.anayltics"; + public const string ExceptionType = "Btms.exception_type"; + public const string MessageType = "Btms.message_type"; } } @@ -39,4 +42,15 @@ internal static DateTime[] CreateDateRange(DateTime from, DateTime to, Aggregati internal static readonly Comparer? byDateTimeResultComparer = Comparer.Create((d1, d2) => d1.Period.CompareTo(d2.Period)); + public static string[] GetImportNotificationSegments() + { + return ModelHelpers.GetChedTypes() + .SelectMany(chedType => new string[] { $"{chedType} Linked", $"{chedType} Not Linked" }) + .ToArray(); + } + + public static string[] GetMovementSegments() + { + return new string[] { "Linked", "Not Linked" }; + } } \ No newline at end of file diff --git a/Btms.Analytics/IImportNotificationsAggregationService.cs b/Btms.Analytics/IImportNotificationsAggregationService.cs index f3c45fb..b768609 100644 --- a/Btms.Analytics/IImportNotificationsAggregationService.cs +++ b/Btms.Analytics/IImportNotificationsAggregationService.cs @@ -2,7 +2,8 @@ namespace Btms.Analytics; public interface IImportNotificationsAggregationService { - public Task ByCreated(DateTime from, DateTime to, AggregationPeriod aggregateBy = AggregationPeriod.Day); - public Task ByArrival(DateTime from, DateTime to, AggregationPeriod aggregateBy = AggregationPeriod.Day); - public Task ByStatus(DateTime from, DateTime to); + public Task ByCreated(DateTime from, DateTime to, AggregationPeriod aggregateBy = AggregationPeriod.Day); + public Task ByArrival(DateTime from, DateTime to, AggregationPeriod aggregateBy = AggregationPeriod.Day); + public Task ByStatus(DateTime from, DateTime to); + public Task ByCommodityCount(DateTime from, DateTime to); } \ No newline at end of file diff --git a/Btms.Analytics/IMovementsAggregationService.cs b/Btms.Analytics/IMovementsAggregationService.cs index 1941289..a3324f0 100644 --- a/Btms.Analytics/IMovementsAggregationService.cs +++ b/Btms.Analytics/IMovementsAggregationService.cs @@ -2,7 +2,10 @@ namespace Btms.Analytics; public interface IMovementsAggregationService { - public Task ByCreated(DateTime from, DateTime to, AggregationPeriod aggregateBy = AggregationPeriod.Day); - public Task ByArrival(DateTime from, DateTime to, AggregationPeriod aggregateBy = AggregationPeriod.Day); - public Task ByStatus(DateTime from, DateTime to); + public Task ByCreated(DateTime from, DateTime to, AggregationPeriod aggregateBy = AggregationPeriod.Day); + public Task ByArrival(DateTime from, DateTime to, AggregationPeriod aggregateBy = AggregationPeriod.Day); + public Task ByStatus(DateTime from, DateTime to); + public Task ByItemCount(DateTime from, DateTime to); + public Task ByUniqueDocumentReferenceCount(DateTime from, DateTime to); + public Task UniqueDocumentReferenceByMovementCount(DateTime from, DateTime to); } \ No newline at end of file diff --git a/Btms.Analytics/ImportNotificationMetrics.cs b/Btms.Analytics/ImportNotificationMetrics.cs index 199ddd0..63b274a 100644 --- a/Btms.Analytics/ImportNotificationMetrics.cs +++ b/Btms.Analytics/ImportNotificationMetrics.cs @@ -4,9 +4,9 @@ using System.Diagnostics.Metrics; using System.Runtime.InteropServices.JavaScript; using System.Text; -using Btms.Analytics.Extensions; using Btms.Common; using Btms.Metrics; +using Btms.Analytics.Extensions; using Btms.Common.Extensions; using Btms.Model; using Btms.Model.Extensions; diff --git a/Btms.Analytics/ImportNotificationsAggregationService.cs b/Btms.Analytics/ImportNotificationsAggregationService.cs index 34be1e8..67345d9 100644 --- a/Btms.Analytics/ImportNotificationsAggregationService.cs +++ b/Btms.Analytics/ImportNotificationsAggregationService.cs @@ -1,7 +1,6 @@ using System.Collections; using Microsoft.Extensions.Logging; using System.Linq.Expressions; -using Btms.Analytics.Extensions; using Btms.Backend.Data; using Btms.Common.Extensions; using Btms.Model.Extensions; @@ -11,11 +10,14 @@ using MongoDB.Bson; using MongoDB.Driver; +using Btms.Analytics.Extensions; +using MongoDB.Driver.Linq; + namespace Btms.Analytics; public class ImportNotificationsAggregationService(IMongoDbContext context, ILogger logger) : IImportNotificationsAggregationService { - public Task ByCreated(DateTime from, DateTime to, AggregationPeriod aggregateBy = AggregationPeriod.Day) + public Task ByCreated(DateTime from, DateTime to, AggregationPeriod aggregateBy = AggregationPeriod.Day) { var dateRange = AnalyticsHelpers.CreateDateRange(from, to, aggregateBy); @@ -28,7 +30,7 @@ string CreateDatasetName(BsonDocument b) => return Aggregate(dateRange, CreateDatasetName, matchFilter, "$createdSource", aggregateBy); } - public Task ByArrival(DateTime from, DateTime to, AggregationPeriod aggregateBy = AggregationPeriod.Day) + public Task ByArrival(DateTime from, DateTime to, AggregationPeriod aggregateBy = AggregationPeriod.Day) { var dateRange = AnalyticsHelpers.CreateDateRange(from, to, aggregateBy); @@ -41,7 +43,7 @@ string CreateDatasetName(BsonDocument b) => return Aggregate(dateRange, CreateDatasetName, matchFilter, "$partOne.arrivesAt", aggregateBy); } - public Task ByStatus(DateTime from, DateTime to) + public Task ByStatus(DateTime from, DateTime to) { var data = context .Notifications @@ -51,20 +53,67 @@ public Task ByStatus(DateTime from, DateTime to) .ToDictionary(g => AnalyticsHelpers.GetLinkedName(g.Linked, g.ImportNotificationType.AsString()!), g => g.Count); - return Task.FromResult(new PieChartDataset() + return Task.FromResult(new SingeSeriesDataset() { - Values = GetSegments().ToDictionary(title => title, title => data.GetValueOrDefault(title, 0)) + Values = AnalyticsHelpers.GetImportNotificationSegments().ToDictionary(title => title, title => data.GetValueOrDefault(title, 0)) }); } - private static string[] GetSegments() + public Task ByCommodityCount(DateTime from, DateTime to) { - return ModelHelpers.GetChedTypes() - .SelectMany(chedType => new string[] { $"{chedType} Linked", $"{chedType} Not Linked" }) - .ToArray(); + var query = context + .Notifications + .Where(n => n.CreatedSource >= from && n.CreatedSource < to) + .GroupBy(n => new + { + ImportNotificationType = n.ImportNotificationType!.Value, + Linked = n.Relationships.Movements.Data.Count > 0, + CommodityCount = n.Commodities.Count() + }) + .Select(g => new { g.Key, Count = g.Count() }); + + var result = query + .Execute(logger) + .GroupBy(r => new { r.Key.ImportNotificationType, r.Key.Linked }) + .ToList(); + + var maxCommodities = result.Max(r => r.Max(i => i.Key.CommodityCount)); + + var list = result + .SelectMany(g => + g.Select(r => + new { + Title = AnalyticsHelpers.GetLinkedName(g.Key.Linked, g.Key.ImportNotificationType.AsString()!), + r.Key.CommodityCount, + NotificationCount = r.Count + }) + ) + .ToList(); + + var asDictionary = list + .ToDictionary( + g => new { g.Title, g.CommodityCount }, + g => g.NotificationCount); + + + return Task.FromResult( + AnalyticsHelpers.GetImportNotificationSegments() + .Select(title => new MultiSeriesDataset(title, "ItemCount") + { + // Results = asDictionary.AsResultList(title, maxCommodities) + Results = Enumerable.Range(0, maxCommodities) + .Select(i => new ByNumericDimensionResult() + { + Dimension = i, + Value = asDictionary.GetValueOrDefault(new { Title=title, CommodityCount = i }) + }).ToList() + } + ) + .AsOrderedArray(d => d.Name) + ); } - private Task Aggregate(DateTime[] dateRange, Func createDatasetName, Expression> filter, string dateField, AggregationPeriod aggregateBy) + private Task Aggregate(DateTime[] dateRange, Func createDatasetName, Expression> filter, string dateField, AggregationPeriod aggregateBy) { var truncateBy = aggregateBy == AggregationPeriod.Hour ? "hour" : "day"; @@ -80,9 +129,9 @@ private Task Aggregate(DateTime[] dateRange, Func mongoResult.AsDataset(dateRange, title)) - .AsOrderedArray(); + .AsOrderedArray(d => d.Name); logger.LogDebug("Aggregated Data {result}", output.ToList().ToJsonString()); diff --git a/Btms.Analytics/MovementsAggregationService.cs b/Btms.Analytics/MovementsAggregationService.cs index 0f35dce..ec2799e 100644 --- a/Btms.Analytics/MovementsAggregationService.cs +++ b/Btms.Analytics/MovementsAggregationService.cs @@ -1,7 +1,8 @@ using System.Collections; +using System.Data.Common; using Microsoft.Extensions.Logging; using System.Linq.Expressions; -using Btms.Analytics.Extensions; +using System.Text.RegularExpressions; using Btms.Backend.Data; using Btms.Common.Extensions; using Btms.Model.Extensions; @@ -11,6 +12,8 @@ using MongoDB.Bson; using MongoDB.Driver; +using Btms.Analytics.Extensions; + namespace Btms.Analytics; public class MovementsAggregationService(IMongoDbContext context, ILogger logger) : IMovementsAggregationService @@ -23,7 +26,7 @@ public class MovementsAggregationService(IMongoDbContext context, ILoggerTime period to search to (exclusive) /// Aggregate by day/hour /// - public Task ByCreated(DateTime from, DateTime to, AggregationPeriod aggregateBy = AggregationPeriod.Day) + public Task ByCreated(DateTime from, DateTime to, AggregationPeriod aggregateBy = AggregationPeriod.Day) { var dateRange = AnalyticsHelpers.CreateDateRange(from, to, aggregateBy); @@ -42,7 +45,7 @@ public Task ByCreated(DateTime from, DateTime to, AggregationPeriod a /// Time period to search to (exclusive) /// Aggregate by day/hour /// - public Task ByArrival(DateTime from, DateTime to, AggregationPeriod aggregateBy = AggregationPeriod.Day) + public Task ByArrival(DateTime from, DateTime to, AggregationPeriod aggregateBy = AggregationPeriod.Day) { var dateRange = AnalyticsHelpers.CreateDateRange(from, to, aggregateBy); @@ -54,7 +57,7 @@ public Task ByArrival(DateTime from, DateTime to, AggregationPeriod a return Aggregate(dateRange, CreateDatasetName, matchFilter, "$arrivesAt", aggregateBy); } - public Task ByStatus(DateTime from, DateTime to) + public Task ByStatus(DateTime from, DateTime to) { var data = context .Movements @@ -63,13 +66,120 @@ public Task ByStatus(DateTime from, DateTime to) .Select(g => new { g.Key, Count = g.Count() }) .ToDictionary(g => AnalyticsHelpers.GetLinkedName(g.Key), g => g.Count); - return Task.FromResult(new PieChartDataset() + return Task.FromResult(new SingeSeriesDataset() { - Values = new string[] { "Linked", "Not Linked" }.ToDictionary(title => title, title => data.GetValueOrDefault(title, 0)) + Values = AnalyticsHelpers.GetMovementSegments().ToDictionary(title => title, title => data.GetValueOrDefault(title, 0)) }); } + + public Task ByItemCount(DateTime from, DateTime to) + { + var mongoQuery = context + .Movements + .Where(n => n.CreatedSource >= from && n.CreatedSource < to) + .GroupBy(m => new { Linked = m.Relationships.Notifications.Data.Count > 0, Items = m.Items.Count }) + .GroupBy(g => g.Key.Linked); + + var mongoResult = mongoQuery + .Execute(logger) + .SelectMany(g => g.Select(l => new + { + Linked = g.Key, + ItemCount = l.Key.Items, + Count = l.Count() + })) + .ToList(); + + var dictionary = mongoResult + .ToDictionary(g => new { Title = AnalyticsHelpers.GetLinkedName(g.Linked), ItemCount = g.ItemCount }, g => g.Count); + + var maxCount = mongoResult + .Max(r => r.Count); + + return Task.FromResult(AnalyticsHelpers.GetMovementSegments() + .Select(title => new MultiSeriesDataset(title, "Item Count") { + Results = Enumerable.Range(0, maxCount + 1) + .Select(i => new ByNumericDimensionResult() + { + Dimension = i, + Value = dictionary!.GetValueOrDefault(new { Title=title, ItemCount = i }, 0) + }).ToList() + }) + .ToArray() + ); + } - private Task Aggregate(DateTime[] dateRange, Func createDatasetName, Expression> filter, string dateField, AggregationPeriod aggregateBy) + public Task ByUniqueDocumentReferenceCount(DateTime from, DateTime to) + { + var mongoQuery = context + .Movements + .Where(n => n.CreatedSource >= from && n.CreatedSource < to) + .GroupBy(m => new + { + Linked = m.Relationships.Notifications.Data.Count > 0, + DocumentReferenceCount = m.Items + .SelectMany(i => i.Documents == null ? new string[] {} : i.Documents.Select(d => d.DocumentReference)) + .Distinct() + .Count() + }) + .GroupBy(g => g.Key.Linked); + + var mongoResult = mongoQuery + .Execute(logger) + .SelectMany(g => g.Select(l => new + { + Linked = g.Key, + DocumentReferenceCount = l.Key.DocumentReferenceCount, + MovementCount = l.Count() + })) + .ToList(); + + var dictionary = mongoResult + .ToDictionary( + g => new { Title = AnalyticsHelpers.GetLinkedName(g.Linked), DocumentReferenceCount = g.DocumentReferenceCount }, + g => g.MovementCount)!; + + var maxReferences = mongoResult.Max(r => r.DocumentReferenceCount); + + return Task.FromResult(AnalyticsHelpers.GetMovementSegments() + .Select(title => new MultiSeriesDataset(title, "Document Reference Count") { + Results = Enumerable.Range(0, maxReferences + 1) + .Select(i => new ByNumericDimensionResult() + { + Dimension = i, + Value = dictionary!.GetValueOrDefault(new { Title=title, DocumentReferenceCount = i }, 0) + }).ToList() + }) + .ToArray() + ); + } + + public Task UniqueDocumentReferenceByMovementCount(DateTime from, DateTime to) + { + var mongoQuery = context + .Movements + .Where(m => m.CreatedSource >= from && m.CreatedSource < to) + .SelectMany(m => m.Items.Select(i => new { Item = i, MovementId = m.Id })) + .SelectMany(i => i.Item.Documents!.Select(d => + new { MovementId = i.MovementId, DocumentReference = d.DocumentReference })) + .Distinct() + .GroupBy(d => d.DocumentReference) + .Select(d => new { DocumentReference = d.Key, MovementCount = d.Count() }) + .GroupBy(d => d.MovementCount) + .Select(d => new { MovementCount = d.Key, DocumentReferenceCount = d.Count() }); + + var mongoResult = mongoQuery + .Execute(logger) + .ToDictionary( + r =>r.MovementCount.ToString(), + r=> r.DocumentReferenceCount); + + var result = new SingeSeriesDataset() { Values = mongoResult }; + + return Task.FromResult(result); + } + + private Task Aggregate(DateTime[] dateRange, Func createDatasetName, Expression> filter, string dateField, AggregationPeriod aggregateBy) { var truncateBy = aggregateBy == AggregationPeriod.Hour ? "hour" : "day"; @@ -85,9 +195,9 @@ private Task Aggregate(DateTime[] dateRange, Func mongoResult.AsDataset(dateRange, title)) - .AsOrderedArray(); + .AsOrderedArray(m => m.Name); logger.LogDebug("Aggregated Data {result}", output.ToList().ToJsonString()); diff --git a/Btms.Analytics/Results.cs b/Btms.Analytics/Results.cs new file mode 100644 index 0000000..1411594 --- /dev/null +++ b/Btms.Analytics/Results.cs @@ -0,0 +1,31 @@ +namespace Btms.Analytics; + +public class ByDateTimeResult +{ + public DateTime Period { get; set; } + public int Value { get; set; } +} + +public class ByNumericDimensionResult +{ + public int Dimension { get; set; } + public int Value { get; set; } +} + +public class SingeSeriesDataset +{ + public IDictionary Values { get; set; } = new Dictionary(); +} + +public class MultiSeriesDatetimeDataset(string name) +{ + public string Name { get; set; } = name; + public List Periods { get; set; } = []; +} + +public class MultiSeriesDataset(string name, string dimension) +{ + public string Name { get; set; } = name; + public string Dimension { get; set; } = dimension; + public List Results { get; set; } = []; +} \ No newline at end of file diff --git a/Btms.Backend/Endpoints/AnalyticsEndpoints.cs b/Btms.Backend/Endpoints/AnalyticsEndpoints.cs index c16fa80..deb8967 100644 --- a/Btms.Backend/Endpoints/AnalyticsEndpoints.cs +++ b/Btms.Backend/Endpoints/AnalyticsEndpoints.cs @@ -20,47 +20,69 @@ private static async Task RecordCurrentState( await importNotificationMetrics.RecordCurrentState(); return Results.Ok(); } + private static async Task GetDashboard( [FromServices] IImportNotificationsAggregationService importService, [FromServices] IMovementsAggregationService movementsService) { var importNotificationLinkingByCreated = await importService .ByCreated(DateTime.Today.MonthAgo(), DateTime.Today); - + var importNotificationLinkingByArrival = await importService - .ByArrival(DateTime.Today.MonthAgo(), DateTime.Today.MonthLater()) ; - + .ByArrival(DateTime.Today.MonthAgo(), DateTime.Today.MonthLater()); + var last7DaysImportNotificationsLinkingStatus = await importService .ByStatus(DateTime.Today.WeekAgo(), DateTime.Now); - + var last24HoursImportNotificationsLinkingStatus = await importService .ByStatus(DateTime.Now.Yesterday(), DateTime.Now); - - var last24HoursImportNotificationsLinkingByCreated= await importService + + var last24HoursImportNotificationsLinkingByCreated = await importService .ByCreated(DateTime.Now.NextHour().Yesterday(), DateTime.Now.NextHour(), AggregationPeriod.Hour); - + var lastMonthImportNotificationsByTypeAndStatus = await importService .ByStatus(DateTime.Today.MonthAgo(), DateTime.Now); - + var last24HoursMovementsLinkingByCreated = await movementsService .ByCreated(DateTime.Now.NextHour().Yesterday(), DateTime.Now.NextHour(), AggregationPeriod.Hour); - + var movementsLinkingByCreated = await movementsService - .ByCreated(DateTime.Today.MonthAgo(), DateTime.Today) ; - + .ByCreated(DateTime.Today.MonthAgo(), DateTime.Today); + var movementsLinkingByArrival = await movementsService .ByArrival(DateTime.Today.MonthAgo(), DateTime.Today.MonthLater()); - + var lastMonthMovementsByStatus = await movementsService .ByStatus(DateTime.Today.MonthAgo(), DateTime.Now); - + + var lastMonthMovementsByItemCount = await movementsService + .ByItemCount(DateTime.Today.MonthAgo(), DateTime.Now); + + var lastMonthMovementsByUniqueDocumentReferenceCount = await movementsService + .ByUniqueDocumentReferenceCount(DateTime.Today.MonthAgo(), DateTime.Now); + + var lastMonthUniqueDocumentReferenceByMovementCount = await movementsService + .UniqueDocumentReferenceByMovementCount(DateTime.Today.MonthAgo(), DateTime.Now); + + var lastMonthImportNotificationsByCommodityCount = await importService + .ByCommodityCount(DateTime.Today.MonthAgo(), DateTime.Now); + return Results.Ok(new { - importNotificationLinkingByCreated, importNotificationLinkingByArrival, - last7DaysImportNotificationsLinkingStatus, last24HoursImportNotificationsLinkingStatus, - last24HoursMovementsLinkingByCreated, last24HoursImportNotificationsLinkingByCreated, - movementsLinkingByCreated, movementsLinkingByArrival, - lastMonthImportNotificationsByTypeAndStatus, lastMonthMovementsByStatus + importNotificationLinkingByCreated, + importNotificationLinkingByArrival, + last7DaysImportNotificationsLinkingStatus, + last24HoursImportNotificationsLinkingStatus, + last24HoursMovementsLinkingByCreated, + last24HoursImportNotificationsLinkingByCreated, + movementsLinkingByCreated, + movementsLinkingByArrival, + lastMonthImportNotificationsByTypeAndStatus, + lastMonthMovementsByStatus, + lastMonthMovementsByItemCount, + lastMonthImportNotificationsByCommodityCount, + lastMonthMovementsByUniqueDocumentReferenceCount, + lastMonthUniqueDocumentReferenceByMovementCount }); } } \ No newline at end of file diff --git a/TestDataGenerator/ClearanceRequestBuilder.cs b/TestDataGenerator/ClearanceRequestBuilder.cs index 5e1529d..c51cac5 100644 --- a/TestDataGenerator/ClearanceRequestBuilder.cs +++ b/TestDataGenerator/ClearanceRequestBuilder.cs @@ -67,12 +67,26 @@ public ClearanceRequestBuilder WithArrivalDateTimeOffset(DateOnly? date, Time public ClearanceRequestBuilder WithItem(string documentCode, string commodityCode, string description, int netWeight) { - return Do(x => + return Do(cr => { - x.Items![0].TaricCommodityCode = commodityCode; - x.Items![0].GoodsDescription = description; - x.Items![0].ItemNetMass = netWeight; - x.Items![0].Documents![0].DocumentCode = documentCode; + cr.Items![0].TaricCommodityCode = commodityCode; + cr.Items![0].GoodsDescription = description; + cr.Items![0].ItemNetMass = netWeight; + cr.Items![0].Documents![0].DocumentCode = documentCode; + }); + } + + public ClearanceRequestBuilder WithRandomItems(int min, int max) + { + var commodityCount = CreateRandomInt(min, max); + + return Do(cr => + { + var items = Enumerable.Range(0, commodityCount) + .Select(_ => cr.Items![0]) + .ToArray(); + + cr.Items = items; }); } diff --git a/TestDataGenerator/Helpers/BuilderExtensions.cs b/TestDataGenerator/Helpers/BuilderExtensions.cs index 304ab70..ad0dc2e 100644 --- a/TestDataGenerator/Helpers/BuilderExtensions.cs +++ b/TestDataGenerator/Helpers/BuilderExtensions.cs @@ -16,7 +16,7 @@ public static IServiceCollection ConfigureTestGenerationServices(this IServiceCo services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); - services.AddSingleton(); + services.AddSingleton(); var blobOptionsValidatorDescriptor = services.SingleOrDefault(d => d.ServiceType == typeof(IValidateOptions))!; services.Remove(blobOptionsValidatorDescriptor); diff --git a/TestDataGenerator/Program.cs b/TestDataGenerator/Program.cs index 25553ae..97ef1a3 100644 --- a/TestDataGenerator/Program.cs +++ b/TestDataGenerator/Program.cs @@ -101,7 +101,8 @@ private static async Task Main(string[] args) { app.CreateScenarioConfig(5, 7), app.CreateScenarioConfig(5, 7), - app.CreateScenarioConfig(15, 7) + app.CreateScenarioConfig(15, 7), + app.CreateScenarioConfig(15, 7) } }, new diff --git a/TestDataGenerator/README.md b/TestDataGenerator/README.md index ed87313..8486651 100644 --- a/TestDataGenerator/README.md +++ b/TestDataGenerator/README.md @@ -1,4 +1,4 @@ -# CDMS Test Data Generator +# Btms Test Data Generator This test generator allows us to manage sets of test data for different uses, and store them, either in blob storage or locally. diff --git a/TestDataGenerator/Scenarios/CRNoMatchScenarioGenerator.cs b/TestDataGenerator/Scenarios/CRNoMatchScenarioGenerator.cs index a9acb48..5d15240 100644 --- a/TestDataGenerator/Scenarios/CRNoMatchScenarioGenerator.cs +++ b/TestDataGenerator/Scenarios/CRNoMatchScenarioGenerator.cs @@ -5,7 +5,7 @@ namespace TestDataGenerator.Scenarios; -public class CRNoMatchScenarioGenerator(ILogger logger) : ScenarioGenerator +public class CrNoMatchScenarioGenerator(ILogger logger) : ScenarioGenerator { public override GeneratorResult Generate(int scenario, int item, DateTime entryDate, ScenarioConfig config) { @@ -13,6 +13,7 @@ public override GeneratorResult Generate(int scenario, int item, DateTime entryD .WithEntryDate(entryDate) .WithArrivalDateTimeOffset(DateTime.Today.ToDate(), DateTime.Now.ToTime()) .WithReferenceNumber(DataHelpers.GenerateReferenceNumber(ImportNotificationTypeEnum.Cveda, scenario, entryDate, item)) + .WithRandomItems(10, 100) .ValidateAndBuild(); logger.LogInformation("Created {EntryReference}", clearanceRequest.Header!.EntryReference);