From 1b759f5ac75f41e0f62aa23ad6a85a3118dcdf81 Mon Sep 17 00:00:00 2001 From: Craig Edmunds Date: Fri, 13 Dec 2024 16:43:56 +0000 Subject: [PATCH 1/5] CDMS-200 refactoring to make decision analytics easier --- .../Helpers/MultiSeriesDatasetAssertions.cs | 4 +- .../Helpers/TestAssertionExtensions.cs | 2 +- .../ImportNotificationsByArrivalDateTests.cs | 1 + .../ImportNotificationsByCommoditiesTests.cs | 1 + .../ImportNotificationsByCreatedDateTests.cs | 3 + .../MovementsByCreatedDateTests.cs | 3 + Btms.Analytics.Tests/MovementsByItemsTests.cs | 1 + ...MovementsByUniqueDocumentReferenceTests.cs | 1 + .../Extensions/AnalyticsExtensions.cs | 18 +-- .../IImportNotificationsAggregationService.cs | 6 +- .../IMovementsAggregationService.cs | 7 +- Btms.Analytics/ImportNotificationMetrics.cs | 6 +- .../ImportNotificationsAggregationService.cs | 50 ++++++-- Btms.Analytics/MovementsAggregationService.cs | 67 ++++++---- Btms.Analytics/Results.cs | 35 +++++- Btms.Backend/Config/AnalyticsDashboards.cs | 119 ++++++++++++++++++ Btms.Backend/Endpoints/AnalyticsEndpoints.cs | 76 +++-------- Btms.Backend/Endpoints/SyncEndpoints.cs | 15 +-- Btms.Backend/Program.cs | 4 + TestDataGenerator/README.md | 12 +- 20 files changed, 300 insertions(+), 131 deletions(-) create mode 100644 Btms.Backend/Config/AnalyticsDashboards.cs diff --git a/Btms.Analytics.Tests/Helpers/MultiSeriesDatasetAssertions.cs b/Btms.Analytics.Tests/Helpers/MultiSeriesDatasetAssertions.cs index 85350c7b..5c1df55a 100644 --- a/Btms.Analytics.Tests/Helpers/MultiSeriesDatasetAssertions.cs +++ b/Btms.Analytics.Tests/Helpers/MultiSeriesDatasetAssertions.cs @@ -3,8 +3,8 @@ namespace Btms.Analytics.Tests.Helpers; -public class MultiSeriesDatasetAssertions(List? test) - : GenericCollectionAssertions(test) +public class MultiSeriesDatasetAssertions(List? test) + : GenericCollectionAssertions(test) { [CustomAssertion] public void BeSameLength(string because = "", params object[] becauseArgs) diff --git a/Btms.Analytics.Tests/Helpers/TestAssertionExtensions.cs b/Btms.Analytics.Tests/Helpers/TestAssertionExtensions.cs index 2fbd2248..4838c8a8 100644 --- a/Btms.Analytics.Tests/Helpers/TestAssertionExtensions.cs +++ b/Btms.Analytics.Tests/Helpers/TestAssertionExtensions.cs @@ -2,7 +2,7 @@ namespace Btms.Analytics.Tests.Helpers; public static class TestAssertionExtensions { - public static MultiSeriesDatasetAssertions Should(this List? instance) + public static MultiSeriesDatasetAssertions Should(this List? instance) { return new MultiSeriesDatasetAssertions(instance); } diff --git a/Btms.Analytics.Tests/ImportNotificationsByArrivalDateTests.cs b/Btms.Analytics.Tests/ImportNotificationsByArrivalDateTests.cs index cfc4a535..61c9bafb 100644 --- a/Btms.Analytics.Tests/ImportNotificationsByArrivalDateTests.cs +++ b/Btms.Analytics.Tests/ImportNotificationsByArrivalDateTests.cs @@ -20,6 +20,7 @@ public async Task WhenCalledNextMonth_ReturnExpectedAggregation() var result = (await basicSampleDataTestFixture.ImportNotificationsAggregationService .ByArrival(DateTime.Today, DateTime.Today.MonthLater())) + .Series .ToList(); testOutputHelper.WriteLine($"{result.Count} aggregated items found"); diff --git a/Btms.Analytics.Tests/ImportNotificationsByCommoditiesTests.cs b/Btms.Analytics.Tests/ImportNotificationsByCommoditiesTests.cs index c132682f..10ce1dcf 100644 --- a/Btms.Analytics.Tests/ImportNotificationsByCommoditiesTests.cs +++ b/Btms.Analytics.Tests/ImportNotificationsByCommoditiesTests.cs @@ -20,6 +20,7 @@ public async Task WhenCalledLastWeek_ReturnExpectedAggregation() testOutputHelper.WriteLine("Querying for aggregated data"); var result = (await multiItemDataTestFixture.ImportNotificationsAggregationService .ByCommodityCount(DateTime.Today.WeekAgo(), DateTime.Today.Tomorrow())) + .Series .ToList(); testOutputHelper.WriteLine("{0} aggregated items found", result.Count); diff --git a/Btms.Analytics.Tests/ImportNotificationsByCreatedDateTests.cs b/Btms.Analytics.Tests/ImportNotificationsByCreatedDateTests.cs index 8426b5fb..de4e3f1f 100644 --- a/Btms.Analytics.Tests/ImportNotificationsByCreatedDateTests.cs +++ b/Btms.Analytics.Tests/ImportNotificationsByCreatedDateTests.cs @@ -18,6 +18,7 @@ public async Task WhenCalledLast48Hours_ReturnExpectedAggregation() { var result = (await basicSampleDataTestFixture.ImportNotificationsAggregationService .ByCreated(DateTime.Now.NextHour().AddDays(-2), DateTime.Now.NextHour(),AggregationPeriod.Hour)) + .Series .ToList(); testOutputHelper.WriteLine(result.ToJsonString()); @@ -36,6 +37,7 @@ public async Task WhenCalledLastMonth_ReturnExpectedAggregation() { var result = (await basicSampleDataTestFixture.ImportNotificationsAggregationService .ByCreated(DateTime.Today.MonthAgo(), DateTime.Today.Tomorrow())) + .Series .ToList(); testOutputHelper.WriteLine(result.ToJsonString()); @@ -63,6 +65,7 @@ public async Task WhenCalledWithTimePeriodYieldingNoResults_ReturnEmptyAggregati var result = (await basicSampleDataTestFixture.ImportNotificationsAggregationService .ByCreated(from, to, AggregationPeriod.Hour)) + .Series .ToList(); testOutputHelper.WriteLine(result.ToJsonString()); diff --git a/Btms.Analytics.Tests/MovementsByCreatedDateTests.cs b/Btms.Analytics.Tests/MovementsByCreatedDateTests.cs index a4905180..e0173b34 100644 --- a/Btms.Analytics.Tests/MovementsByCreatedDateTests.cs +++ b/Btms.Analytics.Tests/MovementsByCreatedDateTests.cs @@ -17,6 +17,7 @@ public async Task WhenCalledLast48Hours_ReturnExpectedAggregation() { var result = (await basicSampleDataTestFixture.MovementsAggregationService .ByCreated(DateTime.Now.NextHour().AddDays(-2), DateTime.Now.NextHour(), AggregationPeriod.Hour)) + .Series .ToList(); testOutputHelper.WriteLine(result.ToJsonString()); @@ -38,6 +39,7 @@ public async Task WhenCalledWithTimePeriodYieldingNoResults_ReturnEmptyAggregati var result = (await basicSampleDataTestFixture.MovementsAggregationService .ByCreated(from, to, AggregationPeriod.Hour)) + .Series .ToList(); testOutputHelper.WriteLine(result.ToJsonString()); @@ -62,6 +64,7 @@ public async Task WhenCalledLastMonth_ReturnExpectedAggregation() { var result = (await basicSampleDataTestFixture.MovementsAggregationService .ByCreated(DateTime.Today.MonthAgo(), DateTime.Today.Tomorrow())) + .Series .ToList(); testOutputHelper.WriteLine(result.ToJsonString()); diff --git a/Btms.Analytics.Tests/MovementsByItemsTests.cs b/Btms.Analytics.Tests/MovementsByItemsTests.cs index 1c2d9ce2..477367a6 100644 --- a/Btms.Analytics.Tests/MovementsByItemsTests.cs +++ b/Btms.Analytics.Tests/MovementsByItemsTests.cs @@ -20,6 +20,7 @@ public async Task WhenCalledLastWeek_ReturnExpectedAggregation() testOutputHelper.WriteLine("Querying for aggregated data"); var result = (await multiItemDataTestFixture.MovementsAggregationService .ByItemCount(DateTime.Today.WeekAgo(), DateTime.Today.Tomorrow())) + .Series .ToList(); testOutputHelper.WriteLine("{0} aggregated items found", result.Count); diff --git a/Btms.Analytics.Tests/MovementsByUniqueDocumentReferenceTests.cs b/Btms.Analytics.Tests/MovementsByUniqueDocumentReferenceTests.cs index b0b80be2..52c61f3f 100644 --- a/Btms.Analytics.Tests/MovementsByUniqueDocumentReferenceTests.cs +++ b/Btms.Analytics.Tests/MovementsByUniqueDocumentReferenceTests.cs @@ -20,6 +20,7 @@ public async Task WhenCalledLastWeek_ReturnExpectedAggregation() testOutputHelper.WriteLine("Querying for aggregated data"); var result = (await multiItemDataTestFixture.MovementsAggregationService .ByUniqueDocumentReferenceCount(DateTime.Today.WeekAgo(), DateTime.Today.Tomorrow())) + .Series .ToList(); testOutputHelper.WriteLine("{0} aggregated items found", result.Count); diff --git a/Btms.Analytics/Extensions/AnalyticsExtensions.cs b/Btms.Analytics/Extensions/AnalyticsExtensions.cs index de14f73c..d567be39 100644 --- a/Btms.Analytics/Extensions/AnalyticsExtensions.cs +++ b/Btms.Analytics/Extensions/AnalyticsExtensions.cs @@ -28,7 +28,7 @@ public static IServiceCollection AddAnalyticsServices(this IServiceCollection se return services; } - public static string MetricsKey(this MultiSeriesDatetimeDataset ds) + public static string MetricsKey(this DatetimeSeries ds) { return ds.Name.Replace(" ", "-").ToLower(); } @@ -64,17 +64,15 @@ public static Dictionary GetNamedSetAsDict(this Dictionary records, DateTime[] dateRange, string title) + public static DatetimeSeries AsDataset(this Dictionary records, DateTime[] dateRange, + string title) { var dates = records.GetNamedSetAsDict(title); - return new MultiSeriesDatetimeDataset(title) + return new DatetimeSeries(title) { Periods = dateRange .Select(resultDate => - new ByDateTimeResult - { - Period = resultDate, Value = dates.GetValueOrDefault(resultDate, 0) - }) + new ByDateTimeResult { Period = resultDate, Value = dates.GetValueOrDefault(resultDate, 0) }) .Order(AnalyticsHelpers.ByDateTimeResultComparer) .ToList() }; @@ -138,4 +136,10 @@ private static void LogExecutedMongoString(this ILogger logger, IQueryable sourc logger.LogInformation("[{Query}]", string.Join(",", stages.Select(s => s.ToString()).ToArray())); } + + public static Task AsIDataset(this Task ms) + { + ms.Wait(); + return Task.FromResult((IDataset)ms.Result); + } } \ No newline at end of file diff --git a/Btms.Analytics/IImportNotificationsAggregationService.cs b/Btms.Analytics/IImportNotificationsAggregationService.cs index b7686099..11b54d89 100644 --- a/Btms.Analytics/IImportNotificationsAggregationService.cs +++ b/Btms.Analytics/IImportNotificationsAggregationService.cs @@ -2,8 +2,8 @@ namespace Btms.Analytics; public interface IImportNotificationsAggregationService { - public Task ByCreated(DateTime from, DateTime to, AggregationPeriod aggregateBy = AggregationPeriod.Day); - public Task ByArrival(DateTime from, DateTime to, AggregationPeriod aggregateBy = AggregationPeriod.Day); + public Task ByCreated(DateTime from, DateTime to, AggregationPeriod aggregateBy = AggregationPeriod.Day); + public Task ByArrival(DateTime from, DateTime to, AggregationPeriod aggregateBy = AggregationPeriod.Day); public Task ByStatus(DateTime from, DateTime to); - public Task ByCommodityCount(DateTime from, DateTime to); + public Task ByCommodityCount(DateTime from, DateTime to); } \ No newline at end of file diff --git a/Btms.Analytics/IMovementsAggregationService.cs b/Btms.Analytics/IMovementsAggregationService.cs index f9635ae7..20bdecc2 100644 --- a/Btms.Analytics/IMovementsAggregationService.cs +++ b/Btms.Analytics/IMovementsAggregationService.cs @@ -2,9 +2,10 @@ namespace Btms.Analytics; public interface IMovementsAggregationService { - public Task ByCreated(DateTime from, DateTime to, AggregationPeriod aggregateBy = AggregationPeriod.Day); + public Task ByCreated(DateTime from, DateTime to, AggregationPeriod aggregateBy = AggregationPeriod.Day); public Task ByStatus(DateTime from, DateTime to); - public Task ByItemCount(DateTime from, DateTime to); - public Task ByUniqueDocumentReferenceCount(DateTime from, DateTime to); + public Task ByItemCount(DateTime from, DateTime to); + public Task ByUniqueDocumentReferenceCount(DateTime from, DateTime to); public Task UniqueDocumentReferenceByMovementCount(DateTime from, DateTime to); + public Task ByCheck(DateTime from, DateTime to); } \ No newline at end of file diff --git a/Btms.Analytics/ImportNotificationMetrics.cs b/Btms.Analytics/ImportNotificationMetrics.cs index cf839037..d1cb2cbe 100644 --- a/Btms.Analytics/ImportNotificationMetrics.cs +++ b/Btms.Analytics/ImportNotificationMetrics.cs @@ -39,14 +39,14 @@ public async Task RecordCurrentState() { var metrics = await _importService.ByCreated(DateTime.Today, DateTime.Now.NextHour()); - foreach (var dataset in metrics) + foreach (var series in metrics.Series) { - var key = $"{AnalyticsMetricNames.MetricPrefix}.import-notifications.{dataset.MetricsKey()}.count"; + var key = $"{AnalyticsMetricNames.MetricPrefix}.import-notifications.{series.MetricsKey()}.count"; if (_metrics.TryGetValue(key, out var instrument)) { if (instrument is Gauge g) { - g.Record(dataset.Periods[0].Value); + g.Record(series.Periods[0].Value); } else { diff --git a/Btms.Analytics/ImportNotificationsAggregationService.cs b/Btms.Analytics/ImportNotificationsAggregationService.cs index 48f622c0..a6a4a22c 100644 --- a/Btms.Analytics/ImportNotificationsAggregationService.cs +++ b/Btms.Analytics/ImportNotificationsAggregationService.cs @@ -12,7 +12,7 @@ namespace Btms.Analytics; public class ImportNotificationsAggregationService(IMongoDbContext context, ILogger logger) : IImportNotificationsAggregationService { - public Task ByCreated(DateTime from, DateTime to, AggregationPeriod aggregateBy = AggregationPeriod.Day) + public Task ByCreated(DateTime from, DateTime to, AggregationPeriod aggregateBy = AggregationPeriod.Day) { var dateRange = AnalyticsHelpers.CreateDateRange(from, to, aggregateBy); @@ -25,7 +25,7 @@ string CreateDatasetName(BsonDocument b) => return Aggregate(dateRange, CreateDatasetName, matchFilter, "$createdSource", aggregateBy); } - public Task ByArrival(DateTime from, DateTime to, AggregationPeriod aggregateBy = AggregationPeriod.Day) + public Task ByArrival(DateTime from, DateTime to, AggregationPeriod aggregateBy = AggregationPeriod.Day) { var dateRange = AnalyticsHelpers.CreateDateRange(from, to, aggregateBy); @@ -54,7 +54,7 @@ public Task ByStatus(DateTime from, DateTime to) }); } - public Task ByCommodityCount(DateTime from, DateTime to) + public Task ByCommodityCount(DateTime from, DateTime to) { var query = context .Notifications @@ -94,24 +94,48 @@ public Task ByCommodityCount(DateTime from, DateTime to) g => g.NotificationCount); - return Task.FromResult( - AnalyticsHelpers.GetImportNotificationSegments() - .Select(title => new MultiSeriesDataset(title, "ItemCount") + return Task.FromResult(new MultiSeriesDataset() + { + Series = AnalyticsHelpers.GetImportNotificationSegments() + .Select(title => new Series(title, "ItemCount") { - // Results = asDictionary.AsResultList(title, maxCommodities) Results = Enumerable.Range(0, maxCommodities) .Select(i => new ByNumericDimensionResult { Dimension = i, Value = asDictionary.GetValueOrDefault(new { Title=title, CommodityCount = i }) }).ToList() - } - ) - .AsOrderedArray(d => d.Name) - ); + }) + .ToList() + // .AsOrderedArray(d => d.Name) + + }); + + // return Task.FromResult( + // AnalyticsHelpers.GetImportNotificationSegments() + // .Select(title => new MultiSeriesDataset(title, "ItemCount") + // { + // // Results = asDictionary.AsResultList(title, maxCommodities) + // Series = + // [ + // new Series() + // { + // Results = Enumerable.Range(0, maxCommodities) + // .Select(i => new ByNumericDimensionResult + // { + // Dimension = i, + // Value = asDictionary.GetValueOrDefault(new { Title=title, CommodityCount = i }) + // }).ToList() + // } + // ] + // + // } + // ) + // .AsOrderedArray(d => d.Name) + // ); } - private Task Aggregate(DateTime[] dateRange, Func createDatasetName, Expression> filter, string dateField, AggregationPeriod aggregateBy) + private Task Aggregate(DateTime[] dateRange, Func createDatasetName, Expression> filter, string dateField, AggregationPeriod aggregateBy) { var truncateBy = aggregateBy == AggregationPeriod.Hour ? "hour" : "day"; @@ -133,6 +157,6 @@ private Task Aggregate(DateTime[] dateRange, Func< logger.LogDebug("Aggregated Data {Result}", output.ToList().ToJsonString()); - return Task.FromResult(output); + return Task.FromResult(new MultiSeriesDatetimeDataset() { Series = output.ToList() }); } } \ No newline at end of file diff --git a/Btms.Analytics/MovementsAggregationService.cs b/Btms.Analytics/MovementsAggregationService.cs index 387da2a2..82356796 100644 --- a/Btms.Analytics/MovementsAggregationService.cs +++ b/Btms.Analytics/MovementsAggregationService.cs @@ -18,7 +18,7 @@ public class MovementsAggregationService(IMongoDbContext context, ILoggerTime period to search to (exclusive) /// Aggregate by day/hour /// - public Task ByCreated(DateTime from, DateTime to, AggregationPeriod aggregateBy = AggregationPeriod.Day) + public Task ByCreated(DateTime from, DateTime to, AggregationPeriod aggregateBy = AggregationPeriod.Day) { var dateRange = AnalyticsHelpers.CreateDateRange(from, to, aggregateBy); @@ -45,7 +45,7 @@ public Task ByStatus(DateTime from, DateTime to) }); } - public Task ByItemCount(DateTime from, DateTime to) + public Task ByItemCount(DateTime from, DateTime to) { var mongoQuery = context .Movements @@ -63,20 +63,24 @@ public Task ByItemCount(DateTime from, DateTime to) var maxCount = mongoResult.Count > 0 ? mongoResult.Max(r => r.Count) : 0; - return Task.FromResult(AnalyticsHelpers.GetMovementSegments() - .Select(title => new MultiSeriesDataset(title, "Item Count") { - Results = Enumerable.Range(0, maxCount + 1) - .Select(i => new ByNumericDimensionResult + return Task.FromResult(new MultiSeriesDataset() + { + Series = AnalyticsHelpers.GetMovementSegments() + .Select(title => new Series(title, "Item Count") { - Dimension = i, - Value = dictionary.GetValueOrDefault(new { Title=title, ItemCount = i }, 0) - }).ToList() - }) - .ToArray() - ); + Results = Enumerable.Range(0, maxCount + 1) + .Select(i => new ByNumericDimensionResult + { + Dimension = i, + Value = dictionary.GetValueOrDefault(new { Title=title, ItemCount = i }, 0) + }).ToList() + } + ) + .ToList() + }); } - public Task ByUniqueDocumentReferenceCount(DateTime from, DateTime to) + public Task ByUniqueDocumentReferenceCount(DateTime from, DateTime to) { var mongoQuery = context .Movements @@ -100,18 +104,22 @@ public Task ByUniqueDocumentReferenceCount(DateTime from, var maxReferences = mongoResult.Count > 0 ? mongoResult.Max(r => r.DocumentReferenceCount) : 0; - - return Task.FromResult(AnalyticsHelpers.GetMovementSegments() - .Select(title => new MultiSeriesDataset(title, "Document Reference Count") { - Results = Enumerable.Range(0, maxReferences + 1) - .Select(i => new ByNumericDimensionResult - { - Dimension = i, - Value = dictionary.GetValueOrDefault(new { Title=title, DocumentReferenceCount = i }, 0) - }).ToList() - }) - .ToArray() - ); + + return Task.FromResult(new MultiSeriesDataset() + { + Series = AnalyticsHelpers.GetMovementSegments() + .Select(title => new Series(title, "Document Reference Count") + { + Results = Enumerable.Range(0, maxReferences + 1) + .Select(i => new ByNumericDimensionResult + { + Dimension = i, + Value = dictionary.GetValueOrDefault(new { Title = title, DocumentReferenceCount = i }, + 0) + }).ToList() + }) + .ToList() + }); } public Task UniqueDocumentReferenceByMovementCount(DateTime from, DateTime to) @@ -139,7 +147,12 @@ public Task UniqueDocumentReferenceByMovementCount(DateTime return Task.FromResult(result); } - private Task Aggregate(DateTime[] dateRange, Func createDatasetName, Expression> filter, string dateField, AggregationPeriod aggregateBy) + public Task ByCheck(DateTime from, DateTime to) + { + return Task.FromResult(new MultiSeriesDataset() ); + } + + private Task Aggregate(DateTime[] dateRange, Func createDatasetName, Expression> filter, string dateField, AggregationPeriod aggregateBy) { var truncateBy = aggregateBy == AggregationPeriod.Hour ? "hour" : "day"; @@ -161,6 +174,6 @@ private Task Aggregate(DateTime[] dateRange, Func< logger.LogDebug("Aggregated Data {Result}", output.ToList().ToJsonString()); - return Task.FromResult(output); + return Task.FromResult(new MultiSeriesDatetimeDataset() { Series = output.ToList() }); } } \ No newline at end of file diff --git a/Btms.Analytics/Results.cs b/Btms.Analytics/Results.cs index 14115947..5085d84d 100644 --- a/Btms.Analytics/Results.cs +++ b/Btms.Analytics/Results.cs @@ -1,3 +1,7 @@ +using System.Diagnostics.CodeAnalysis; +using System.Text.Json; +using System.Text.Json.Serialization; + namespace Btms.Analytics; public class ByDateTimeResult @@ -12,18 +16,43 @@ public class ByNumericDimensionResult public int Value { get; set; } } -public class SingeSeriesDataset +public class SingeSeriesDataset : IDataset { public IDictionary Values { get; set; } = new Dictionary(); } -public class MultiSeriesDatetimeDataset(string name) +public class TypeMappingConverter : JsonConverter + where TImplementation : TType +{ + [return: MaybeNull] + public override TType Read( + ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) => + JsonSerializer.Deserialize(ref reader, options); + + public override void Write( + Utf8JsonWriter writer, TType value, JsonSerializerOptions options) => + JsonSerializer.Serialize(writer, (TImplementation)value!, options); +} + +public interface IDataset; + +public class MultiSeriesDatetimeDataset : IDataset +{ + public List Series { get; set; } = []; +} + +public class DatetimeSeries(string name) { public string Name { get; set; } = name; public List Periods { get; set; } = []; } -public class MultiSeriesDataset(string name, string dimension) +public class MultiSeriesDataset : IDataset +{ + public List Series { get; set; } = []; +} + +public class Series(string name, string dimension) { public string Name { get; set; } = name; public string Dimension { get; set; } = dimension; diff --git a/Btms.Backend/Config/AnalyticsDashboards.cs b/Btms.Backend/Config/AnalyticsDashboards.cs new file mode 100644 index 00000000..83141874 --- /dev/null +++ b/Btms.Backend/Config/AnalyticsDashboards.cs @@ -0,0 +1,119 @@ +using Btms.Analytics; +using Btms.Analytics.Extensions; +using Btms.Common.Extensions; +using FluentAssertions; +using MongoDB.Driver.Linq; + +namespace Btms.Backend.Config; + +public static class AnalyticsDashboards +{ + public static async Task> GetCharts( + ILogger logger, + IImportNotificationsAggregationService importService, + IMovementsAggregationService movementsService, + string[] chartsToRender) + { + var charts = new Dictionary>> + { + { + "importNotificationLinkingByCreated", + () => importService.ByCreated(DateTime.Today.MonthAgo(), DateTime.Today).AsIDataset() + }, + // { + // "importNotificationLinkingByArrival", + // () => importService.ByArrival(DateTime.Today.MonthAgo(), DateTime.Today.MonthLater()) + // } + }; + // + // var chartsToReturn = chartsToRender.Length == 0 + // ? charts + // : charts.Where(keyValuePair => chartsToRender.Contains(keyValuePair.Key)); + // + // var tasks = chartsToReturn + // .Select(c => c.Value()); + + // var results = await Task.WhenAll(tasks.ToArray()); + // .ToList() + + var f = charts["importNotificationLinkingByCreated"]; + var importNotificationLinkingByCreated = await f(); + + logger.LogInformation("Results found {0}, {1} Series", importNotificationLinkingByCreated, ((MultiSeriesDatetimeDataset)importNotificationLinkingByCreated).Series.Count); + return new Dictionary() + { + { "importNotificationLinkingByCreated", importNotificationLinkingByCreated } + }; + // return new Dictionary() + // { + // { "importNotificationLinkingByCreated", importNotificationLinkingByCreated } + // }; + // return new { importNotificationLinkingByCreated, importNotificationLinkingByCreated }; + // return await Task.FromResult(Results.Ok( + // importNotificationLinkingByCreated + // )); + + // return await Task.FromResult(Results.Ok(new { + // importNotificationLinkingByCreated + // })); + // var chartTasks = + // var output = Parallel.ForEachAsync(chartsToReturn, async (keyValuePair, token) => await keyValuePair.Value()); + // var output = Parallel.ForEach(chartsToReturn, (keyValuePair, token) => keyValuePair.Value()); + // return await Task.Run(() => //Parallel.ForEach(chartsToReturn, () => + // { + // Console.WriteLine("Testing"); + // })); + + // return await Task.Run(() => Console.WriteLine("Testing")); + + // return Task.WaitAll(chartsToReturn.Select(keyValuePair => keyValuePair.Value())); + // foreach (var keyValuePair in chartsToReturn) + // { + // if (!(chartsToRender.Length == 0) || chartsToRender.Contains(keyValuePair.Key)) + // { + // var result = keyValuePair.Value(); + // } + // } + + // return output.ToList(); + + // var importNotificationLinkingByCreated = await importService + // .ByCreated(DateTime.Today.MonthAgo(), DateTime.Today); + // + // var importNotificationLinkingByArrival = await importService + // .ByArrival(DateTime.Today.MonthAgo(), DateTime.Today.MonthLater()); + + // var last7DaysImportNotificationsLinkingStatus = await importService + // .ByStatus(DateTime.Today.WeekAgo(), DateTime.Now); + // + // var last24HoursImportNotificationsLinkingStatus = await importService + // .ByStatus(DateTime.Now.Yesterday(), DateTime.Now); + // + // var last24HoursImportNotificationsLinkingByCreated = await importService + // .ByCreated(DateTime.Now.NextHour().Yesterday(), DateTime.Now.NextHour(), AggregationPeriod.Hour); + // + // var lastMonthImportNotificationsByTypeAndStatus = await importService + // .ByStatus(DateTime.Today.MonthAgo(), DateTime.Now); + // + // var last24HoursMovementsLinkingByCreated = await movementsService + // .ByCreated(DateTime.Now.NextHour().Yesterday(), DateTime.Now.NextHour(), AggregationPeriod.Hour); + // + // var movementsLinkingByCreated = await movementsService + // .ByCreated(DateTime.Today.MonthAgo(), DateTime.Today); + // + // var lastMonthMovementsByStatus = await movementsService + // .ByStatus(DateTime.Today.MonthAgo(), DateTime.Now); + // + // var lastMonthMovementsByItemCount = await movementsService + // .ByItemCount(DateTime.Today.MonthAgo(), DateTime.Now); + // + // var lastMonthMovementsByUniqueDocumentReferenceCount = await movementsService + // .ByUniqueDocumentReferenceCount(DateTime.Today.MonthAgo(), DateTime.Now); + // + // var lastMonthUniqueDocumentReferenceByMovementCount = await movementsService + // .UniqueDocumentReferenceByMovementCount(DateTime.Today.MonthAgo(), DateTime.Now); + // + // var lastMonthImportNotificationsByCommodityCount = await importService + // .ByCommodityCount(DateTime.Today.MonthAgo(), DateTime.Now); + } +} \ No newline at end of file diff --git a/Btms.Backend/Endpoints/AnalyticsEndpoints.cs b/Btms.Backend/Endpoints/AnalyticsEndpoints.cs index cd3cb951..3ae2a934 100644 --- a/Btms.Backend/Endpoints/AnalyticsEndpoints.cs +++ b/Btms.Backend/Endpoints/AnalyticsEndpoints.cs @@ -1,5 +1,9 @@ +using System.Text.Json; using Btms.Analytics; +using Btms.Backend.Config; +using Btms.Common; using Btms.Common.Extensions; +using Btms.Model.Extensions; using Microsoft.AspNetCore.Mvc; namespace Btms.Backend.Endpoints; @@ -22,62 +26,22 @@ private static async Task RecordCurrentState( private static async Task GetDashboard( [FromServices] IImportNotificationsAggregationService importService, - [FromServices] IMovementsAggregationService movementsService) + [FromServices] IMovementsAggregationService movementsService, + [FromQuery] string[] chartsToRender) { - var importNotificationLinkingByCreated = await importService - .ByCreated(DateTime.Today.MonthAgo(), DateTime.Today); - - var importNotificationLinkingByArrival = await importService - .ByArrival(DateTime.Today.MonthAgo(), DateTime.Today.MonthLater()); - - var last7DaysImportNotificationsLinkingStatus = await importService - .ByStatus(DateTime.Today.WeekAgo(), DateTime.Now); - - var last24HoursImportNotificationsLinkingStatus = await importService - .ByStatus(DateTime.Now.Yesterday(), DateTime.Now); - - var last24HoursImportNotificationsLinkingByCreated = await importService - .ByCreated(DateTime.Now.NextHour().Yesterday(), DateTime.Now.NextHour(), AggregationPeriod.Hour); - - var lastMonthImportNotificationsByTypeAndStatus = await importService - .ByStatus(DateTime.Today.MonthAgo(), DateTime.Now); - - var last24HoursMovementsLinkingByCreated = await movementsService - .ByCreated(DateTime.Now.NextHour().Yesterday(), DateTime.Now.NextHour(), AggregationPeriod.Hour); - - var movementsLinkingByCreated = await movementsService - .ByCreated(DateTime.Today.MonthAgo(), DateTime.Today); - - var lastMonthMovementsByStatus = await movementsService - .ByStatus(DateTime.Today.MonthAgo(), DateTime.Now); - - var lastMonthMovementsByItemCount = await movementsService - .ByItemCount(DateTime.Today.MonthAgo(), DateTime.Now); - - var lastMonthMovementsByUniqueDocumentReferenceCount = await movementsService - .ByUniqueDocumentReferenceCount(DateTime.Today.MonthAgo(), DateTime.Now); - - var lastMonthUniqueDocumentReferenceByMovementCount = await movementsService - .UniqueDocumentReferenceByMovementCount(DateTime.Today.MonthAgo(), DateTime.Now); - - var lastMonthImportNotificationsByCommodityCount = await importService - .ByCommodityCount(DateTime.Today.MonthAgo(), DateTime.Now); - - return Results.Ok(new - { - importNotificationLinkingByCreated, - importNotificationLinkingByArrival, - last7DaysImportNotificationsLinkingStatus, - last24HoursImportNotificationsLinkingStatus, - last24HoursMovementsLinkingByCreated, - last24HoursImportNotificationsLinkingByCreated, - movementsLinkingByCreated, - lastMonthMovementsByStatus, - lastMonthMovementsByItemCount, - lastMonthImportNotificationsByCommodityCount, - lastMonthMovementsByUniqueDocumentReferenceCount, - lastMonthImportNotificationsByTypeAndStatus, - lastMonthUniqueDocumentReferenceByMovementCount - }); + var logger = ApplicationLogging.CreateLogger("AnalyticsEndpoints"); + var result = await AnalyticsDashboards.GetCharts(logger, importService, movementsService, chartsToRender); + + var options = + new JsonSerializerOptions + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + Converters = + { + new TypeMappingConverter() + } + }; + + return TypedResults.Json(result, options); } } \ No newline at end of file diff --git a/Btms.Backend/Endpoints/SyncEndpoints.cs b/Btms.Backend/Endpoints/SyncEndpoints.cs index 56ef602b..90727a26 100644 --- a/Btms.Backend/Endpoints/SyncEndpoints.cs +++ b/Btms.Backend/Endpoints/SyncEndpoints.cs @@ -27,6 +27,7 @@ public static void UseSyncEndpoints(this IEndpointRouteBuilder app, IOptions SyncGmrs([FromServices] IBtmsMediator mediato return Results.Accepted($"/sync/jobs/{command.JobId}", command.JobId); } - ////private static async Task GetSyncDecisions( - //// [FromServices] IBtmsMediator mediator, - //// SyncPeriod syncPeriod) - ////{ - //// SyncDecisionsCommand command = new() { SyncPeriod = syncPeriod }; - //// return await SyncDecisions(mediator, command); - ////} + private static async Task GetSyncDecisions( + [FromServices] IBtmsMediator mediator, + SyncPeriod syncPeriod) + { + SyncDecisionsCommand command = new() { SyncPeriod = syncPeriod }; + return await SyncDecisions(mediator, command); + } private static async Task SyncDecisions([FromServices] IBtmsMediator mediator, [FromBody] SyncDecisionsCommand command) diff --git a/Btms.Backend/Program.cs b/Btms.Backend/Program.cs index ab96634a..b1d0663d 100644 --- a/Btms.Backend/Program.cs +++ b/Btms.Backend/Program.cs @@ -38,6 +38,7 @@ using Btms.Azure.Extensions; using Environment = System.Environment; using Btms.Backend.Asb; +using Btms.Common; //-------- Configure the WebApplication builder------------------// @@ -229,6 +230,9 @@ static WebApplication BuildWebApplication(WebApplicationBuilder builder) { var app = builder.Build(); + // Allows us to make a global logger factory available for use where we can't get it from DI, e.g. from static functions + ApplicationLogging.LoggerFactory = app.Services.GetService(); + app.UseEmfExporter(); app.UseAuthentication(); app.UseAuthorization(); diff --git a/TestDataGenerator/README.md b/TestDataGenerator/README.md index 114c115f..66411f65 100644 --- a/TestDataGenerator/README.md +++ b/TestDataGenerator/README.md @@ -13,15 +13,15 @@ isn't quite working, so I've been generating locally and then syncing to blob st az account set --subscription 7d775166-9d6c-4ac2-91a5-61904bae5caa -az storage blob directory delete --container-name dmp-data-1001 --directory-path GENERATED-LOADTEST --account-name -snddmpinfdl1001 --recursive -az storage blob directory delete --container-name dmp-data-1001 --directory-path GENERATED-LOADTEST-BASIC --account-name -snddmpinfdl1001 --recursive +az storage blob directory delete --container-name dmp-data-1001 --directory-path GENERATED-LOADTEST --account-name snddmpinfdl1001 --recursive +az storage blob directory delete --container-name dmp-data-1001 --directory-path GENERATED-LOADTEST-BASIC --account-name snddmpinfdl1001 --recursive +az storage blob directory delete --container-name dmp-data-1001 --directory-path PRODREDACTED-20241204 --account-name snddmpinfdl1001 --recursive az storage blob directory upload -c dmp-data-1001 -d --account-name snddmpinfdl1001 -s TestDataGenerator/.test-data-generator/GENERATED-LOADTEST --recursive az storage blob upload-batch -d dmp-data-1001 --account-name snddmpinfdl1001 -s TestDataGenerator/.test-data-generator/GENERATED-LOADTEST-90Dx10k --destination-path GENERATED-LOADTEST-90Dx10k -~~~~az storage blob upload-batch -d dmp-data-1001 --account-name snddmpinfdl1001 -s -TestDataGenerator/.test-data-generator/GENERATED-LOADTEST-BASIC --destination-path GENERATED-LOADTEST-BASIC \ No newline at end of file + +az storage blob upload-batch -d dmp-data-1001 --account-name snddmpinfdl1001 -s TestDataGenerator/.test-data-generator/GENERATED-LOADTEST-BASIC --destination-path GENERATED-LOADTEST-BASIC +az storage blob upload-batch -d dmp-data-1001 --account-name snddmpinfdl1001 -s TestDataGenerator/.test-data-generator/PRODREDACTED-20241204 --destination-path PRODREDACTED-20241204 \ No newline at end of file From ce27576eb72b65812184dae6d5b6cc11d771ff14 Mon Sep 17 00:00:00 2001 From: Craig Edmunds Date: Mon, 16 Dec 2024 17:25:32 +0000 Subject: [PATCH 2/5] CDMS-200 refactors analytics chart mechanism to run in parrallel and allow charts to be specified --- .../Helpers/SingleSeriesDatasetAssertions.cs | 2 +- .../Helpers/TestAssertionExtensions.cs | 2 +- .../Extensions/AnalyticsExtensions.cs | 12 ++ .../IImportNotificationsAggregationService.cs | 2 +- .../IMovementsAggregationService.cs | 4 +- .../ImportNotificationsAggregationService.cs | 4 +- Btms.Analytics/MovementsAggregationService.cs | 8 +- Btms.Analytics/Results.cs | 42 +++-- .../AnalyticsTests.cs | 105 ++++++++++++ .../Extensions/HttpExtensions.cs | 14 ++ Btms.Backend/Config/AnalyticsDashboards.cs | 157 ++++++++---------- Btms.Backend/Endpoints/AnalyticsEndpoints.cs | 4 +- Btms.Common/Extensions/StringExtensions.cs | 20 +++ 13 files changed, 262 insertions(+), 114 deletions(-) create mode 100644 Btms.Backend.IntegrationTests/AnalyticsTests.cs create mode 100644 Btms.Backend.IntegrationTests/Extensions/HttpExtensions.cs create mode 100644 Btms.Common/Extensions/StringExtensions.cs diff --git a/Btms.Analytics.Tests/Helpers/SingleSeriesDatasetAssertions.cs b/Btms.Analytics.Tests/Helpers/SingleSeriesDatasetAssertions.cs index e8ff273d..dbc212b7 100644 --- a/Btms.Analytics.Tests/Helpers/SingleSeriesDatasetAssertions.cs +++ b/Btms.Analytics.Tests/Helpers/SingleSeriesDatasetAssertions.cs @@ -2,7 +2,7 @@ namespace Btms.Analytics.Tests.Helpers; -public class SingleSeriesDatasetAssertions(SingeSeriesDataset? test) +public class SingleSeriesDatasetAssertions(SingleSeriesDataset? test) { [CustomAssertion] public void HaveResults(string because = "", params object[] becauseArgs) diff --git a/Btms.Analytics.Tests/Helpers/TestAssertionExtensions.cs b/Btms.Analytics.Tests/Helpers/TestAssertionExtensions.cs index 4838c8a8..3e21166f 100644 --- a/Btms.Analytics.Tests/Helpers/TestAssertionExtensions.cs +++ b/Btms.Analytics.Tests/Helpers/TestAssertionExtensions.cs @@ -6,7 +6,7 @@ public static MultiSeriesDatasetAssertions Should(this List? instance) { return new MultiSeriesDatasetAssertions(instance); } - public static SingleSeriesDatasetAssertions Should(this SingeSeriesDataset? instance) + public static SingleSeriesDatasetAssertions Should(this SingleSeriesDataset? instance) { return new SingleSeriesDatasetAssertions(instance); } diff --git a/Btms.Analytics/Extensions/AnalyticsExtensions.cs b/Btms.Analytics/Extensions/AnalyticsExtensions.cs index d567be39..2d7bbd6d 100644 --- a/Btms.Analytics/Extensions/AnalyticsExtensions.cs +++ b/Btms.Analytics/Extensions/AnalyticsExtensions.cs @@ -142,4 +142,16 @@ public static Task AsIDataset(this Task ms ms.Wait(); return Task.FromResult((IDataset)ms.Result); } + + public static Task AsIDataset(this Task ms) + { + ms.Wait(); + return Task.FromResult((IDataset)ms.Result); + } + + public static Task AsIDataset(this Task ms) + { + ms.Wait(); + return Task.FromResult((IDataset)ms.Result); + } } \ No newline at end of file diff --git a/Btms.Analytics/IImportNotificationsAggregationService.cs b/Btms.Analytics/IImportNotificationsAggregationService.cs index 11b54d89..a6b40638 100644 --- a/Btms.Analytics/IImportNotificationsAggregationService.cs +++ b/Btms.Analytics/IImportNotificationsAggregationService.cs @@ -4,6 +4,6 @@ public interface IImportNotificationsAggregationService { public Task ByCreated(DateTime from, DateTime to, AggregationPeriod aggregateBy = AggregationPeriod.Day); public Task ByArrival(DateTime from, DateTime to, AggregationPeriod aggregateBy = AggregationPeriod.Day); - public Task ByStatus(DateTime from, DateTime to); + public Task ByStatus(DateTime from, DateTime to); public Task ByCommodityCount(DateTime from, DateTime to); } \ No newline at end of file diff --git a/Btms.Analytics/IMovementsAggregationService.cs b/Btms.Analytics/IMovementsAggregationService.cs index 20bdecc2..a54925ca 100644 --- a/Btms.Analytics/IMovementsAggregationService.cs +++ b/Btms.Analytics/IMovementsAggregationService.cs @@ -3,9 +3,9 @@ namespace Btms.Analytics; public interface IMovementsAggregationService { public Task ByCreated(DateTime from, DateTime to, AggregationPeriod aggregateBy = AggregationPeriod.Day); - public Task ByStatus(DateTime from, DateTime to); + public Task ByStatus(DateTime from, DateTime to); public Task ByItemCount(DateTime from, DateTime to); public Task ByUniqueDocumentReferenceCount(DateTime from, DateTime to); - public Task UniqueDocumentReferenceByMovementCount(DateTime from, DateTime to); + public Task UniqueDocumentReferenceByMovementCount(DateTime from, DateTime to); public Task ByCheck(DateTime from, DateTime to); } \ No newline at end of file diff --git a/Btms.Analytics/ImportNotificationsAggregationService.cs b/Btms.Analytics/ImportNotificationsAggregationService.cs index a6a4a22c..6dfde720 100644 --- a/Btms.Analytics/ImportNotificationsAggregationService.cs +++ b/Btms.Analytics/ImportNotificationsAggregationService.cs @@ -38,7 +38,7 @@ string CreateDatasetName(BsonDocument b) => return Aggregate(dateRange, CreateDatasetName, matchFilter, "$partOne.arrivesAt", aggregateBy); } - public Task ByStatus(DateTime from, DateTime to) + public Task ByStatus(DateTime from, DateTime to) { var data = context .Notifications @@ -48,7 +48,7 @@ public Task ByStatus(DateTime from, DateTime to) .ToDictionary(g => AnalyticsHelpers.GetLinkedName(g.Linked, g.ImportNotificationType.AsString()), g => g.Count); - return Task.FromResult(new SingeSeriesDataset + return Task.FromResult(new SingleSeriesDataset { Values = AnalyticsHelpers.GetImportNotificationSegments().ToDictionary(title => title, title => data.GetValueOrDefault(title, 0)) }); diff --git a/Btms.Analytics/MovementsAggregationService.cs b/Btms.Analytics/MovementsAggregationService.cs index 82356796..db4ba94a 100644 --- a/Btms.Analytics/MovementsAggregationService.cs +++ b/Btms.Analytics/MovementsAggregationService.cs @@ -30,7 +30,7 @@ public Task ByCreated(DateTime from, DateTime to, Ag return Aggregate(dateRange, CreateDatasetName, matchFilter, "$createdSource", aggregateBy); } - public Task ByStatus(DateTime from, DateTime to) + public Task ByStatus(DateTime from, DateTime to) { var data = context .Movements @@ -39,7 +39,7 @@ public Task ByStatus(DateTime from, DateTime to) .Select(g => new { g.Key, Count = g.Count() }) .ToDictionary(g => AnalyticsHelpers.GetLinkedName(g.Key), g => g.Count); - return Task.FromResult(new SingeSeriesDataset + return Task.FromResult(new SingleSeriesDataset { Values = AnalyticsHelpers.GetMovementSegments().ToDictionary(title => title, title => data.GetValueOrDefault(title, 0)) }); @@ -122,7 +122,7 @@ public Task ByUniqueDocumentReferenceCount(DateTime from, Da }); } - public Task UniqueDocumentReferenceByMovementCount(DateTime from, DateTime to) + public Task UniqueDocumentReferenceByMovementCount(DateTime from, DateTime to) { var mongoQuery = context .Movements @@ -142,7 +142,7 @@ public Task UniqueDocumentReferenceByMovementCount(DateTime r =>r.MovementCount.ToString(), r=> r.DocumentReferenceCount); - var result = new SingeSeriesDataset { Values = mongoResult }; + var result = new SingleSeriesDataset { Values = mongoResult }; return Task.FromResult(result); } diff --git a/Btms.Analytics/Results.cs b/Btms.Analytics/Results.cs index 5085d84d..0c2eac95 100644 --- a/Btms.Analytics/Results.cs +++ b/Btms.Analytics/Results.cs @@ -16,26 +16,46 @@ public class ByNumericDimensionResult public int Value { get; set; } } -public class SingeSeriesDataset : IDataset -{ - public IDictionary Values { get; set; } = new Dictionary(); -} - -public class TypeMappingConverter : JsonConverter - where TImplementation : TType +/// +/// Serialise the derived types of IDataset +/// +/// +public class ResultTypeMappingConverter : JsonConverter where TType : IDataset { [return: MaybeNull] public override TType Read( ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) => - JsonSerializer.Deserialize(ref reader, options); + throw new NotImplementedException(); - public override void Write( - Utf8JsonWriter writer, TType value, JsonSerializerOptions options) => - JsonSerializer.Serialize(writer, (TImplementation)value!, options); + public override void Write(Utf8JsonWriter writer, TType value, JsonSerializerOptions options) + { + if (value is MultiSeriesDatetimeDataset) + { + JsonSerializer.Serialize(writer, value as MultiSeriesDatetimeDataset, options); + } + else if (value is MultiSeriesDataset) + { + JsonSerializer.Serialize(writer, value as MultiSeriesDataset, options); + } + else if (value is SingleSeriesDataset) + { + JsonSerializer.Serialize(writer, value as SingleSeriesDataset, options); + } + else + { + throw new NotImplementedException(); + } + } } +// A marker interface to identify things we want to be able to return from the analytics API public interface IDataset; +public class SingleSeriesDataset : IDataset +{ + public IDictionary Values { get; set; } = new Dictionary(); +} + public class MultiSeriesDatetimeDataset : IDataset { public List Series { get; set; } = []; diff --git a/Btms.Backend.IntegrationTests/AnalyticsTests.cs b/Btms.Backend.IntegrationTests/AnalyticsTests.cs new file mode 100644 index 00000000..1cb9d985 --- /dev/null +++ b/Btms.Backend.IntegrationTests/AnalyticsTests.cs @@ -0,0 +1,105 @@ +using System.Diagnostics.CodeAnalysis; +using Btms.Common.Extensions; +using Btms.Model; +using Btms.SyncJob; +using Btms.Backend.IntegrationTests.JsonApiClient; +using FluentAssertions; +using System.Net; +using System.Net.Http.Json; +using System.Text; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Text.Json.Serialization; +using Btms.Backend.IntegrationTests.Extensions; +using Btms.Backend.IntegrationTests.Helpers; +using Json.More; +using Xunit; +using Xunit.Abstractions; + +namespace Btms.Backend.IntegrationTests; + +[Trait("Category", "Integration")] +public class AnalyticsTests(IntegrationTestsApplicationFactory factory, ITestOutputHelper testOutputHelper) + : BaseApiTests(factory, testOutputHelper), IClassFixture +{ + + // private static void ShouldNotBeNull([DoesNotReturnIf(true), NotNull] T? value) + // { + // // throw if null + // value.Should().NotBeNull(); + // } + + [Fact] + public async Task GetIndividualMultiSeriesDatetimeChart() + { + //Act + var response = await Client.GetAsync( + "/analytics/dashboard?chartsToRender=importNotificationLinkingByCreated"); + + // Assert + response.IsSuccessStatusCode.Should().BeTrue(response.StatusCode.ToString()); + + var responseDictionary = await response.ToJsonDictionary(); + + responseDictionary.Count.Should().Be(1); + + responseDictionary.Keys.Should().Equal("importNotificationLinkingByCreated"); + + } + + [Fact] + public async Task GetIndividualMultiSeriesChart() + { + //Act + var response = await Client.GetAsync( + "/analytics/dashboard?chartsToRender=lastMonthMovementsByItemCount"); + + // Assert + response.IsSuccessStatusCode.Should().BeTrue(response.StatusCode.ToString()); + + var responseDictionary = await response.ToJsonDictionary(); + + responseDictionary.Count.Should().Be(1); + + responseDictionary.Keys.Should().Equal("lastMonthMovementsByItemCount"); + + } + + [Fact] + public async Task GetIndividualSingleSeriesChart() + { + //Act + var response = await Client.GetAsync( + "/analytics/dashboard?chartsToRender=last7DaysImportNotificationsLinkingStatus"); + + // Assert + response.IsSuccessStatusCode.Should().BeTrue(response.StatusCode.ToString()); + + var responseDictionary = await response.ToJsonDictionary(); + + responseDictionary.Count.Should().Be(1); + + responseDictionary.Keys.Should().Equal("last7DaysImportNotificationsLinkingStatus"); + + } + + [Fact] + public async Task GetAllCharts() + { + //Act + var response = await Client.GetAsync("/analytics/dashboard"); + + // Assert + response.IsSuccessStatusCode.Should().BeTrue(response.StatusCode.ToString()); + + var responseDictionary = await response.ToJsonDictionary(); + + responseDictionary.Count.Should().Be(13); + + responseDictionary.Keys.Take(2).Should().Equal( + "importNotificationLinkingByCreated", + "importNotificationLinkingByArrival"); + + responseDictionary["importNotificationLinkingByCreated"].Should().NotBeNull(); + } +} \ No newline at end of file diff --git a/Btms.Backend.IntegrationTests/Extensions/HttpExtensions.cs b/Btms.Backend.IntegrationTests/Extensions/HttpExtensions.cs new file mode 100644 index 00000000..0c4ec614 --- /dev/null +++ b/Btms.Backend.IntegrationTests/Extensions/HttpExtensions.cs @@ -0,0 +1,14 @@ +using System.Text.Json.Nodes; +using Btms.Common.Extensions; + +namespace Btms.Backend.IntegrationTests.Extensions; + +public static class HttpExtensions +{ + public static async Task> ToJsonDictionary(this HttpResponseMessage? response) + { + var responseDictionary = (await response!.Content.ReadAsStringAsync()).ToJsonDictionary(); + ArgumentNullException.ThrowIfNull(responseDictionary); + return responseDictionary; + } +} \ No newline at end of file diff --git a/Btms.Backend/Config/AnalyticsDashboards.cs b/Btms.Backend/Config/AnalyticsDashboards.cs index 83141874..452712cd 100644 --- a/Btms.Backend/Config/AnalyticsDashboards.cs +++ b/Btms.Backend/Config/AnalyticsDashboards.cs @@ -20,100 +20,75 @@ public static async Task> GetCharts( "importNotificationLinkingByCreated", () => importService.ByCreated(DateTime.Today.MonthAgo(), DateTime.Today).AsIDataset() }, - // { - // "importNotificationLinkingByArrival", - // () => importService.ByArrival(DateTime.Today.MonthAgo(), DateTime.Today.MonthLater()) - // } - }; - // - // var chartsToReturn = chartsToRender.Length == 0 - // ? charts - // : charts.Where(keyValuePair => chartsToRender.Contains(keyValuePair.Key)); - // - // var tasks = chartsToReturn - // .Select(c => c.Value()); - - // var results = await Task.WhenAll(tasks.ToArray()); - // .ToList() - - var f = charts["importNotificationLinkingByCreated"]; - var importNotificationLinkingByCreated = await f(); - - logger.LogInformation("Results found {0}, {1} Series", importNotificationLinkingByCreated, ((MultiSeriesDatetimeDataset)importNotificationLinkingByCreated).Series.Count); - return new Dictionary() - { - { "importNotificationLinkingByCreated", importNotificationLinkingByCreated } + { + "importNotificationLinkingByArrival", + () => importService.ByArrival(DateTime.Today.MonthAgo(), DateTime.Today.MonthLater()).AsIDataset() + }, + { + "last7DaysImportNotificationsLinkingStatus", + () => importService.ByStatus(DateTime.Today.WeekAgo(), DateTime.Now).AsIDataset() + }, + { + "last24HoursImportNotificationsLinkingStatus", + () => importService.ByStatus(DateTime.Now.Yesterday(), DateTime.Now).AsIDataset() + }, + { + "last24HoursImportNotificationsLinkingByCreated", + () => importService + .ByCreated(DateTime.Now.NextHour().Yesterday(), DateTime.Now.NextHour(), AggregationPeriod.Hour) + .AsIDataset() + }, + { + "lastMonthImportNotificationsByTypeAndStatus", + () => importService.ByStatus(DateTime.Today.MonthAgo(), DateTime.Now).AsIDataset() + }, + { + "last24HoursMovementsLinkingByCreated", + () => movementsService.ByCreated(DateTime.Now.NextHour().Yesterday(), DateTime.Now.NextHour(), AggregationPeriod.Hour).AsIDataset() + }, + { + "movementsLinkingByCreated", + () => movementsService.ByCreated(DateTime.Today.MonthAgo(), DateTime.Today).AsIDataset() + }, + { + "lastMonthMovementsByStatus", + () => movementsService.ByStatus(DateTime.Today.MonthAgo(), DateTime.Now).AsIDataset() + }, + { + "lastMonthMovementsByItemCount", + () => movementsService.ByItemCount(DateTime.Today.MonthAgo(), DateTime.Now).AsIDataset() + }, + { + "lastMonthMovementsByUniqueDocumentReferenceCount", + () => movementsService.ByUniqueDocumentReferenceCount(DateTime.Today.MonthAgo(), DateTime.Now).AsIDataset() + }, + { + "lastMonthUniqueDocumentReferenceByMovementCount", + () => movementsService.UniqueDocumentReferenceByMovementCount(DateTime.Today.MonthAgo(), DateTime.Now).AsIDataset() + }, + { + "lastMonthImportNotificationsByCommodityCount", + () => importService.ByCommodityCount(DateTime.Today.MonthAgo(), DateTime.Now).AsIDataset() + } }; - // return new Dictionary() - // { - // { "importNotificationLinkingByCreated", importNotificationLinkingByCreated } - // }; - // return new { importNotificationLinkingByCreated, importNotificationLinkingByCreated }; - // return await Task.FromResult(Results.Ok( - // importNotificationLinkingByCreated - // )); + + var chartsToReturn = chartsToRender.Length == 0 + ? charts + : charts.Where(keyValuePair => chartsToRender.Contains(keyValuePair.Key)); + + var taskList = chartsToReturn.Select(r => new KeyValuePair>(key:r.Key, value: r.Value())); + + await Task.WhenAll(taskList.Select(r => r.Value)); - // return await Task.FromResult(Results.Ok(new { - // importNotificationLinkingByCreated - // })); - // var chartTasks = - // var output = Parallel.ForEachAsync(chartsToReturn, async (keyValuePair, token) => await keyValuePair.Value()); - // var output = Parallel.ForEach(chartsToReturn, (keyValuePair, token) => keyValuePair.Value()); - // return await Task.Run(() => //Parallel.ForEach(chartsToReturn, () => - // { - // Console.WriteLine("Testing"); - // })); - - // return await Task.Run(() => Console.WriteLine("Testing")); - - // return Task.WaitAll(chartsToReturn.Select(keyValuePair => keyValuePair.Value())); - // foreach (var keyValuePair in chartsToReturn) - // { - // if (!(chartsToRender.Length == 0) || chartsToRender.Contains(keyValuePair.Key)) - // { - // var result = keyValuePair.Value(); - // } - // } + var output = taskList + // .Select(t => t.Value.Result) + .ToDictionary(t => t.Key, t => t.Value.Result); + + logger.LogInformation("Results found {0} Datasets, {1}", output.Count, output.Keys); + return output; + // return output.ToList(); - - // var importNotificationLinkingByCreated = await importService - // .ByCreated(DateTime.Today.MonthAgo(), DateTime.Today); - // - // var importNotificationLinkingByArrival = await importService - // .ByArrival(DateTime.Today.MonthAgo(), DateTime.Today.MonthLater()); - - // var last7DaysImportNotificationsLinkingStatus = await importService - // .ByStatus(DateTime.Today.WeekAgo(), DateTime.Now); - // - // var last24HoursImportNotificationsLinkingStatus = await importService - // .ByStatus(DateTime.Now.Yesterday(), DateTime.Now); - // - // var last24HoursImportNotificationsLinkingByCreated = await importService - // .ByCreated(DateTime.Now.NextHour().Yesterday(), DateTime.Now.NextHour(), AggregationPeriod.Hour); - // - // var lastMonthImportNotificationsByTypeAndStatus = await importService - // .ByStatus(DateTime.Today.MonthAgo(), DateTime.Now); - // - // var last24HoursMovementsLinkingByCreated = await movementsService - // .ByCreated(DateTime.Now.NextHour().Yesterday(), DateTime.Now.NextHour(), AggregationPeriod.Hour); - // - // var movementsLinkingByCreated = await movementsService - // .ByCreated(DateTime.Today.MonthAgo(), DateTime.Today); - // - // var lastMonthMovementsByStatus = await movementsService - // .ByStatus(DateTime.Today.MonthAgo(), DateTime.Now); - // - // var lastMonthMovementsByItemCount = await movementsService - // .ByItemCount(DateTime.Today.MonthAgo(), DateTime.Now); - // - // var lastMonthMovementsByUniqueDocumentReferenceCount = await movementsService - // .ByUniqueDocumentReferenceCount(DateTime.Today.MonthAgo(), DateTime.Now); - // - // var lastMonthUniqueDocumentReferenceByMovementCount = await movementsService - // .UniqueDocumentReferenceByMovementCount(DateTime.Today.MonthAgo(), DateTime.Now); - // - // var lastMonthImportNotificationsByCommodityCount = await importService - // .ByCommodityCount(DateTime.Today.MonthAgo(), DateTime.Now); + } } \ No newline at end of file diff --git a/Btms.Backend/Endpoints/AnalyticsEndpoints.cs b/Btms.Backend/Endpoints/AnalyticsEndpoints.cs index 3ae2a934..3abb90a0 100644 --- a/Btms.Backend/Endpoints/AnalyticsEndpoints.cs +++ b/Btms.Backend/Endpoints/AnalyticsEndpoints.cs @@ -38,7 +38,9 @@ private static async Task GetDashboard( PropertyNamingPolicy = JsonNamingPolicy.CamelCase, Converters = { - new TypeMappingConverter() + new ResultTypeMappingConverter(), + // new ResultTypeMappingConverter(), + // new ResultTypeMappingConverter() } }; diff --git a/Btms.Common/Extensions/StringExtensions.cs b/Btms.Common/Extensions/StringExtensions.cs new file mode 100644 index 00000000..ca9e062e --- /dev/null +++ b/Btms.Common/Extensions/StringExtensions.cs @@ -0,0 +1,20 @@ +using System.Text.Json; +using System.Text.Json.Nodes; + +namespace Btms.Common.Extensions; + +public static class StringExtensions +{ + public static JsonNode? ToJsonNode(this string value) + { + return JsonNode.Parse(value); + } + public static JsonObject? ToJsonObject(this string value) + { + return JsonNode.Parse(value) as JsonObject; + } + public static IDictionary? ToJsonDictionary(this string value) + { + return JsonNode.Parse(value) as IDictionary; + } +} \ No newline at end of file From 49ffb5970d0b1fe22f1f0f2ad0428cb5b26ae9fb Mon Sep 17 00:00:00 2001 From: Craig Edmunds Date: Tue, 17 Dec 2024 12:12:48 +0000 Subject: [PATCH 3/5] CDMS-200 get history of movement from /analytics/history --- Btms.Analytics.Tests/MovementHistoryTests.cs | 28 +++++++++++++++++++ .../Extensions/AnalyticsExtensions.cs | 1 - .../IMovementsAggregationService.cs | 3 ++ Btms.Analytics/MovementsAggregationService.cs | 26 +++++++++++++++++ Btms.Analytics/Results.cs | 13 +++++++++ Btms.Backend/Endpoints/AnalyticsEndpoints.cs | 15 ++++++++-- 6 files changed, 82 insertions(+), 4 deletions(-) create mode 100644 Btms.Analytics.Tests/MovementHistoryTests.cs diff --git a/Btms.Analytics.Tests/MovementHistoryTests.cs b/Btms.Analytics.Tests/MovementHistoryTests.cs new file mode 100644 index 00000000..8f356f78 --- /dev/null +++ b/Btms.Analytics.Tests/MovementHistoryTests.cs @@ -0,0 +1,28 @@ +using Btms.Common.Extensions; +using Xunit; +using Xunit.Abstractions; + +using Btms.Analytics.Tests.Fixtures; +using Btms.Analytics.Tests.Helpers; +using FluentAssertions; + +namespace Btms.Analytics.Tests; + +[Collection(nameof(MultiItemDataTestCollection))] +public class MovementHistoryTests( + MultiItemDataTestFixture multiItemDataTestFixture, + ITestOutputHelper testOutputHelper) +{ + [Fact] + public async Task WhenCalled_ReturnsHistory() + { + testOutputHelper.WriteLine("Querying for history"); + var result = await multiItemDataTestFixture.MovementsAggregationService + .GetHistory("23GB9999001215000001"); + + testOutputHelper.WriteLine("{0} history items found", result.Items.Count()); + + result.Items.Should().HasValue(); + result.Items.Select(a => a.AuditEntry.CreatedSource).Should().BeInAscendingOrder(); + } +} \ No newline at end of file diff --git a/Btms.Analytics/Extensions/AnalyticsExtensions.cs b/Btms.Analytics/Extensions/AnalyticsExtensions.cs index 2d7bbd6d..a9fd1af4 100644 --- a/Btms.Analytics/Extensions/AnalyticsExtensions.cs +++ b/Btms.Analytics/Extensions/AnalyticsExtensions.cs @@ -87,7 +87,6 @@ public static T[] AsOrderedArray(this IEnumerable en, Func internal static IEnumerable> Execute(this IQueryable> source, ILogger logger) { - try { var aggregatedData = source.ToList(); diff --git a/Btms.Analytics/IMovementsAggregationService.cs b/Btms.Analytics/IMovementsAggregationService.cs index a54925ca..cd65edaf 100644 --- a/Btms.Analytics/IMovementsAggregationService.cs +++ b/Btms.Analytics/IMovementsAggregationService.cs @@ -1,3 +1,5 @@ +using Btms.Model.Auditing; + namespace Btms.Analytics; public interface IMovementsAggregationService @@ -8,4 +10,5 @@ public interface IMovementsAggregationService public Task ByUniqueDocumentReferenceCount(DateTime from, DateTime to); public Task UniqueDocumentReferenceByMovementCount(DateTime from, DateTime to); public Task ByCheck(DateTime from, DateTime to); + public Task> GetHistory(string movementId); } \ No newline at end of file diff --git a/Btms.Analytics/MovementsAggregationService.cs b/Btms.Analytics/MovementsAggregationService.cs index db4ba94a..7ce5169a 100644 --- a/Btms.Analytics/MovementsAggregationService.cs +++ b/Btms.Analytics/MovementsAggregationService.cs @@ -4,6 +4,7 @@ using Btms.Backend.Data; using Btms.Model.Extensions; using Btms.Model; +using Btms.Model.Auditing; using MongoDB.Bson; using MongoDB.Driver; @@ -147,6 +148,31 @@ public Task UniqueDocumentReferenceByMovementCount(DateTime return Task.FromResult(result); } + public async Task> GetHistory(string movementId) + { + var movement = await context + .Movements + .Find(movementId); + + var notificationIds = movement!.Relationships.Notifications.Data.Select(n => n.Id); + + var notificationEntries = context.Notifications + .Where(n => notificationIds.Contains(n.Id)) + .SelectMany(n => n.AuditEntries + .Select(a => + new AuditHistory(a, $"ImportNotification", n.Id!) + ) + ); + + var entries = movement!.AuditEntries + .Select(a => new AuditHistory(a, "Movement", movementId)) + .Concat(notificationEntries); + + entries = entries.OrderBy(a => a.AuditEntry.CreatedSource); + + return new EntityDataset(entries); + } + public Task ByCheck(DateTime from, DateTime to) { return Task.FromResult(new MultiSeriesDataset() ); diff --git a/Btms.Analytics/Results.cs b/Btms.Analytics/Results.cs index 0c2eac95..dc228a2f 100644 --- a/Btms.Analytics/Results.cs +++ b/Btms.Analytics/Results.cs @@ -1,6 +1,7 @@ using System.Diagnostics.CodeAnalysis; using System.Text.Json; using System.Text.Json.Serialization; +using Btms.Model.Auditing; namespace Btms.Analytics; @@ -56,6 +57,18 @@ public class SingleSeriesDataset : IDataset public IDictionary Values { get; set; } = new Dictionary(); } +public class AuditHistory(AuditEntry auditEntry, string resourceType, string resourceId) +{ + public AuditEntry AuditEntry { get; set; } = auditEntry; + public string ResourceType { get; set; } = resourceType; + public string ResourceId { get; set; } = resourceId; +} + +public class EntityDataset(IEnumerable items) : IDataset +{ + public IEnumerable Items { get; set; } = items; +} + public class MultiSeriesDatetimeDataset : IDataset { public List Series { get; set; } = []; diff --git a/Btms.Backend/Endpoints/AnalyticsEndpoints.cs b/Btms.Backend/Endpoints/AnalyticsEndpoints.cs index 3abb90a0..cfd7cfa8 100644 --- a/Btms.Backend/Endpoints/AnalyticsEndpoints.cs +++ b/Btms.Backend/Endpoints/AnalyticsEndpoints.cs @@ -16,7 +16,18 @@ public static void UseAnalyticsEndpoints(this IEndpointRouteBuilder app) { app.MapGet(BaseRoute + "/dashboard", GetDashboard).AllowAnonymous(); app.MapGet(BaseRoute + "/record-current-state", RecordCurrentState).AllowAnonymous(); + app.MapGet(BaseRoute + "/timeline", Timeline).AllowAnonymous(); } + private static async Task Timeline( + [FromServices] IImportNotificationsAggregationService importService, + [FromServices] IMovementsAggregationService movementsService, + [FromQuery] string movementId) + { + + + return TypedResults.Json(await movementsService.GetHistory(movementId)); + } + private static async Task RecordCurrentState( [FromServices] ImportNotificationMetrics importNotificationMetrics) { @@ -38,9 +49,7 @@ private static async Task GetDashboard( PropertyNamingPolicy = JsonNamingPolicy.CamelCase, Converters = { - new ResultTypeMappingConverter(), - // new ResultTypeMappingConverter(), - // new ResultTypeMappingConverter() + new ResultTypeMappingConverter() } }; From 0c428727d5996b32890aed4938caac0faeeb3d09 Mon Sep 17 00:00:00 2001 From: Craig Edmunds Date: Tue, 17 Dec 2024 12:17:06 +0000 Subject: [PATCH 4/5] CDMS-200 removes commented out lines --- .../ImportNotificationsAggregationService.cs | 27 ------------------- Btms.Backend/Config/AnalyticsDashboards.cs | 3 --- 2 files changed, 30 deletions(-) diff --git a/Btms.Analytics/ImportNotificationsAggregationService.cs b/Btms.Analytics/ImportNotificationsAggregationService.cs index 6dfde720..26ed8f30 100644 --- a/Btms.Analytics/ImportNotificationsAggregationService.cs +++ b/Btms.Analytics/ImportNotificationsAggregationService.cs @@ -72,8 +72,6 @@ public Task ByCommodityCount(DateTime from, DateTime to) .GroupBy(r => new { r.Key.ImportNotificationType, r.Key.Linked }) .ToList(); - // var maxCommodities = result.Max(r => r.Max(i => i.Key.CommodityCount)); - var maxCommodities = result.Count > 0 ? result.Max(r => r.Any() ? r.Max(i => i.Key.CommodityCount) : 0) : 0; @@ -107,32 +105,7 @@ public Task ByCommodityCount(DateTime from, DateTime to) }).ToList() }) .ToList() - // .AsOrderedArray(d => d.Name) - }); - - // return Task.FromResult( - // AnalyticsHelpers.GetImportNotificationSegments() - // .Select(title => new MultiSeriesDataset(title, "ItemCount") - // { - // // Results = asDictionary.AsResultList(title, maxCommodities) - // Series = - // [ - // new Series() - // { - // Results = Enumerable.Range(0, maxCommodities) - // .Select(i => new ByNumericDimensionResult - // { - // Dimension = i, - // Value = asDictionary.GetValueOrDefault(new { Title=title, CommodityCount = i }) - // }).ToList() - // } - // ] - // - // } - // ) - // .AsOrderedArray(d => d.Name) - // ); } private Task Aggregate(DateTime[] dateRange, Func createDatasetName, Expression> filter, string dateField, AggregationPeriod aggregateBy) diff --git a/Btms.Backend/Config/AnalyticsDashboards.cs b/Btms.Backend/Config/AnalyticsDashboards.cs index 452712cd..df25f6ae 100644 --- a/Btms.Backend/Config/AnalyticsDashboards.cs +++ b/Btms.Backend/Config/AnalyticsDashboards.cs @@ -81,14 +81,11 @@ public static async Task> GetCharts( await Task.WhenAll(taskList.Select(r => r.Value)); var output = taskList - // .Select(t => t.Value.Result) .ToDictionary(t => t.Key, t => t.Value.Result); logger.LogInformation("Results found {0} Datasets, {1}", output.Count, output.Keys); return output; - // return output.ToList(); - } } \ No newline at end of file From 68c9aa0be6549749723dc39ac7e3890f5c9fc1f5 Mon Sep 17 00:00:00 2001 From: Craig Edmunds Date: Tue, 17 Dec 2024 12:22:36 +0000 Subject: [PATCH 5/5] Fixes wait() --- .../Extensions/AnalyticsExtensions.cs | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/Btms.Analytics/Extensions/AnalyticsExtensions.cs b/Btms.Analytics/Extensions/AnalyticsExtensions.cs index a9fd1af4..76ed0998 100644 --- a/Btms.Analytics/Extensions/AnalyticsExtensions.cs +++ b/Btms.Analytics/Extensions/AnalyticsExtensions.cs @@ -136,21 +136,21 @@ private static void LogExecutedMongoString(this ILogger logger, IQueryable sourc logger.LogInformation("[{Query}]", string.Join(",", stages.Select(s => s.ToString()).ToArray())); } - public static Task AsIDataset(this Task ms) + public static async Task AsIDataset(this Task ms) { - ms.Wait(); - return Task.FromResult((IDataset)ms.Result); + await ms; + return (IDataset)ms.Result; } - public static Task AsIDataset(this Task ms) + public static async Task AsIDataset(this Task ms) { - ms.Wait(); - return Task.FromResult((IDataset)ms.Result); + await ms; + return (IDataset)ms.Result; } - public static Task AsIDataset(this Task ms) + public static async Task AsIDataset(this Task ms) { - ms.Wait(); - return Task.FromResult((IDataset)ms.Result); + await ms; + return (IDataset)ms.Result; } } \ No newline at end of file