Feature/cdms 200 decision analytics (#21)

* CDMS-200 refactoring to make decision analytics easier * CDMS-200 refactors analytics chart mechanism to run in parrallel and allow charts to be specified * CDMS-200 get history of movement from /analytics/history * CDMS-200 removes commented out lines * Fixes wait()
DEFRA · Dec 17, 2024 · 53326d0 · 53326d0
1 parent 6a55fc5
commit 53326d0
Show file tree

Hide file tree

Showing 25 changed files with 510 additions and 145 deletions.
diff --git a/Btms.Analytics.Tests/Helpers/MultiSeriesDatasetAssertions.cs b/Btms.Analytics.Tests/Helpers/MultiSeriesDatasetAssertions.cs
@@ -3,8 +3,8 @@
 
 namespace Btms.Analytics.Tests.Helpers;
 
-public class MultiSeriesDatasetAssertions(List<MultiSeriesDataset>? test)
-    : GenericCollectionAssertions<MultiSeriesDataset>(test)
+public class MultiSeriesDatasetAssertions(List<Series>? test)
+    : GenericCollectionAssertions<Series>(test)
 {
     [CustomAssertion]
     public void BeSameLength(string because = "", params object[] becauseArgs)

diff --git a/Btms.Analytics.Tests/Helpers/SingleSeriesDatasetAssertions.cs b/Btms.Analytics.Tests/Helpers/SingleSeriesDatasetAssertions.cs
@@ -2,7 +2,7 @@
 
 namespace Btms.Analytics.Tests.Helpers;
 
-public class SingleSeriesDatasetAssertions(SingeSeriesDataset? test)
+public class SingleSeriesDatasetAssertions(SingleSeriesDataset? test)
 {   
     [CustomAssertion]
     public void HaveResults(string because = "", params object[] becauseArgs)

diff --git a/Btms.Analytics.Tests/Helpers/TestAssertionExtensions.cs b/Btms.Analytics.Tests/Helpers/TestAssertionExtensions.cs
@@ -2,11 +2,11 @@ namespace Btms.Analytics.Tests.Helpers;
 
 public static class TestAssertionExtensions
 {
-    public static MultiSeriesDatasetAssertions Should(this List<MultiSeriesDataset>? instance)
+    public static MultiSeriesDatasetAssertions Should(this List<Series>? instance)
     {
         return new MultiSeriesDatasetAssertions(instance);
     }
-    public static SingleSeriesDatasetAssertions Should(this SingeSeriesDataset? instance)
+    public static SingleSeriesDatasetAssertions Should(this SingleSeriesDataset? instance)
     {
         return new SingleSeriesDatasetAssertions(instance);
     }

diff --git a/Btms.Analytics.Tests/ImportNotificationsByArrivalDateTests.cs b/Btms.Analytics.Tests/ImportNotificationsByArrivalDateTests.cs
@@ -20,6 +20,7 @@ public async Task WhenCalledNextMonth_ReturnExpectedAggregation()
 
         var result = (await basicSampleDataTestFixture.ImportNotificationsAggregationService
             .ByArrival(DateTime.Today, DateTime.Today.MonthLater()))
+            .Series
             .ToList();
 
         testOutputHelper.WriteLine($"{result.Count} aggregated items found");

diff --git a/Btms.Analytics.Tests/ImportNotificationsByCommoditiesTests.cs b/Btms.Analytics.Tests/ImportNotificationsByCommoditiesTests.cs
@@ -20,6 +20,7 @@ public async Task WhenCalledLastWeek_ReturnExpectedAggregation()
         testOutputHelper.WriteLine("Querying for aggregated data");
         var result = (await multiItemDataTestFixture.ImportNotificationsAggregationService
             .ByCommodityCount(DateTime.Today.WeekAgo(), DateTime.Today.Tomorrow()))
+            .Series
             .ToList();
 
         testOutputHelper.WriteLine("{0} aggregated items found", result.Count);

diff --git a/Btms.Analytics.Tests/ImportNotificationsByCreatedDateTests.cs b/Btms.Analytics.Tests/ImportNotificationsByCreatedDateTests.cs
@@ -18,6 +18,7 @@ public async Task WhenCalledLast48Hours_ReturnExpectedAggregation()
     {
         var result = (await basicSampleDataTestFixture.ImportNotificationsAggregationService
             .ByCreated(DateTime.Now.NextHour().AddDays(-2), DateTime.Now.NextHour(),AggregationPeriod.Hour))
+            .Series
             .ToList();
 
         testOutputHelper.WriteLine(result.ToJsonString());
@@ -36,6 +37,7 @@ public async Task WhenCalledLastMonth_ReturnExpectedAggregation()
     {
         var result = (await basicSampleDataTestFixture.ImportNotificationsAggregationService
             .ByCreated(DateTime.Today.MonthAgo(), DateTime.Today.Tomorrow()))
+            .Series
             .ToList();
 
         testOutputHelper.WriteLine(result.ToJsonString());
@@ -63,6 +65,7 @@ public async Task WhenCalledWithTimePeriodYieldingNoResults_ReturnEmptyAggregati
 
         var result = (await basicSampleDataTestFixture.ImportNotificationsAggregationService
                 .ByCreated(from, to, AggregationPeriod.Hour))
+            .Series
             .ToList();
 
         testOutputHelper.WriteLine(result.ToJsonString());

diff --git a/Btms.Analytics.Tests/MovementHistoryTests.cs b/Btms.Analytics.Tests/MovementHistoryTests.cs
@@ -0,0 +1,28 @@
+using Btms.Common.Extensions;
+using Xunit;
+using Xunit.Abstractions;
+
+using Btms.Analytics.Tests.Fixtures;
+using Btms.Analytics.Tests.Helpers;
+using FluentAssertions;
+
+namespace Btms.Analytics.Tests;
+
+[Collection(nameof(MultiItemDataTestCollection))]
+public class MovementHistoryTests(
+    MultiItemDataTestFixture multiItemDataTestFixture,
+    ITestOutputHelper testOutputHelper)
+{
+    [Fact]
+    public async Task WhenCalled_ReturnsHistory()
+    {
+        testOutputHelper.WriteLine("Querying for history");
+        var result = await multiItemDataTestFixture.MovementsAggregationService
+            .GetHistory("23GB9999001215000001");
+
+        testOutputHelper.WriteLine("{0} history items found", result.Items.Count());
+
+        result.Items.Should().HasValue();
+        result.Items.Select(a => a.AuditEntry.CreatedSource).Should().BeInAscendingOrder();
+    }
+}
diff --git a/Btms.Analytics.Tests/MovementsByCreatedDateTests.cs b/Btms.Analytics.Tests/MovementsByCreatedDateTests.cs
@@ -17,6 +17,7 @@ public async Task WhenCalledLast48Hours_ReturnExpectedAggregation()
     {
         var result = (await basicSampleDataTestFixture.MovementsAggregationService
             .ByCreated(DateTime.Now.NextHour().AddDays(-2), DateTime.Now.NextHour(), AggregationPeriod.Hour))
+            .Series
             .ToList();
 
         testOutputHelper.WriteLine(result.ToJsonString());
@@ -38,6 +39,7 @@ public async Task WhenCalledWithTimePeriodYieldingNoResults_ReturnEmptyAggregati
 
         var result = (await basicSampleDataTestFixture.MovementsAggregationService
             .ByCreated(from, to, AggregationPeriod.Hour))
+            .Series
             .ToList();
 
         testOutputHelper.WriteLine(result.ToJsonString());
@@ -62,6 +64,7 @@ public async Task WhenCalledLastMonth_ReturnExpectedAggregation()
     {
         var result = (await basicSampleDataTestFixture.MovementsAggregationService
                 .ByCreated(DateTime.Today.MonthAgo(), DateTime.Today.Tomorrow()))
+            .Series
             .ToList();
 
         testOutputHelper.WriteLine(result.ToJsonString());

diff --git a/Btms.Analytics.Tests/MovementsByItemsTests.cs b/Btms.Analytics.Tests/MovementsByItemsTests.cs
@@ -20,6 +20,7 @@ public async Task WhenCalledLastWeek_ReturnExpectedAggregation()
         testOutputHelper.WriteLine("Querying for aggregated data");
         var result = (await multiItemDataTestFixture.MovementsAggregationService
             .ByItemCount(DateTime.Today.WeekAgo(), DateTime.Today.Tomorrow()))
+            .Series
             .ToList();
 
         testOutputHelper.WriteLine("{0} aggregated items found", result.Count);

diff --git a/Btms.Analytics.Tests/MovementsByUniqueDocumentReferenceTests.cs b/Btms.Analytics.Tests/MovementsByUniqueDocumentReferenceTests.cs
@@ -20,6 +20,7 @@ public async Task WhenCalledLastWeek_ReturnExpectedAggregation()
         testOutputHelper.WriteLine("Querying for aggregated data");
         var result = (await multiItemDataTestFixture.MovementsAggregationService
             .ByUniqueDocumentReferenceCount(DateTime.Today.WeekAgo(), DateTime.Today.Tomorrow()))
+            .Series
             .ToList();
 
         testOutputHelper.WriteLine("{0} aggregated items found", result.Count);

diff --git a/Btms.Analytics/Extensions/AnalyticsExtensions.cs b/Btms.Analytics/Extensions/AnalyticsExtensions.cs
@@ -28,7 +28,7 @@ public static IServiceCollection AddAnalyticsServices(this IServiceCollection se
         return services;
     }
 
-    public static string MetricsKey(this MultiSeriesDatetimeDataset ds)
+    public static string MetricsKey(this DatetimeSeries ds)
     {
         return ds.Name.Replace(" ", "-").ToLower();
     }
@@ -64,17 +64,15 @@ public static Dictionary<DateTime, int> GetNamedSetAsDict(this Dictionary<string
             : [];
     }
 
-    public static MultiSeriesDatetimeDataset AsDataset(this Dictionary<string, BsonDocument> records, DateTime[] dateRange, string title)
+    public static DatetimeSeries AsDataset(this Dictionary<string, BsonDocument> records, DateTime[] dateRange,
+        string title)
     {
         var dates = records.GetNamedSetAsDict(title);
-        return new MultiSeriesDatetimeDataset(title)
+        return new DatetimeSeries(title)
         {
             Periods = dateRange
                 .Select(resultDate =>
-                    new ByDateTimeResult
-                    {
-                        Period = resultDate, Value = dates.GetValueOrDefault(resultDate, 0)
-                    })
+                    new ByDateTimeResult { Period = resultDate, Value = dates.GetValueOrDefault(resultDate, 0) })
                 .Order(AnalyticsHelpers.ByDateTimeResultComparer)
                 .ToList()
         };
@@ -89,7 +87,6 @@ public static T[] AsOrderedArray<T, TKey>(this IEnumerable<T> en, Func<T, TKey>
 
     internal static IEnumerable<IGrouping<TKey, TSource>> Execute<TSource, TKey>(this IQueryable<IGrouping<TKey, TSource>> source, ILogger logger)
     {
-
         try
         {
             var aggregatedData = source.ToList();
@@ -138,4 +135,22 @@ private static void LogExecutedMongoString(this ILogger logger, IQueryable sourc
 
         logger.LogInformation("[{Query}]", string.Join(",", stages.Select(s => s.ToString()).ToArray()));
     }
+
+    public static async Task<IDataset> AsIDataset(this Task<MultiSeriesDatetimeDataset> ms)
+    {
+        await ms;
+        return (IDataset)ms.Result;
+    }
+
+    public static async Task<IDataset> AsIDataset(this Task<MultiSeriesDataset> ms)
+    {
+        await ms;
+        return (IDataset)ms.Result;
+    }
+
+    public static async Task<IDataset> AsIDataset(this Task<SingleSeriesDataset> ms)
+    {
+        await ms;
+        return (IDataset)ms.Result;
+    }
 }
diff --git a/Btms.Analytics/IImportNotificationsAggregationService.cs b/Btms.Analytics/IImportNotificationsAggregationService.cs
@@ -2,8 +2,8 @@ namespace Btms.Analytics;
 
 public interface IImportNotificationsAggregationService
 {
-    public Task<MultiSeriesDatetimeDataset[]> ByCreated(DateTime from, DateTime to, AggregationPeriod aggregateBy = AggregationPeriod.Day);
-    public Task<MultiSeriesDatetimeDataset[]> ByArrival(DateTime from, DateTime to, AggregationPeriod aggregateBy = AggregationPeriod.Day);
-    public Task<SingeSeriesDataset> ByStatus(DateTime from, DateTime to);
-    public Task<MultiSeriesDataset[]> ByCommodityCount(DateTime from, DateTime to);
+    public Task<MultiSeriesDatetimeDataset> ByCreated(DateTime from, DateTime to, AggregationPeriod aggregateBy = AggregationPeriod.Day);
+    public Task<MultiSeriesDatetimeDataset> ByArrival(DateTime from, DateTime to, AggregationPeriod aggregateBy = AggregationPeriod.Day);
+    public Task<SingleSeriesDataset> ByStatus(DateTime from, DateTime to);
+    public Task<MultiSeriesDataset> ByCommodityCount(DateTime from, DateTime to);
 }
diff --git a/Btms.Analytics/IMovementsAggregationService.cs b/Btms.Analytics/IMovementsAggregationService.cs
@@ -1,10 +1,14 @@
+using Btms.Model.Auditing;
+
 namespace Btms.Analytics;
 
 public interface IMovementsAggregationService
 {
-    public Task<MultiSeriesDatetimeDataset[]> ByCreated(DateTime from, DateTime to, AggregationPeriod aggregateBy = AggregationPeriod.Day);
-    public Task<SingeSeriesDataset> ByStatus(DateTime from, DateTime to);
-    public Task<MultiSeriesDataset[]> ByItemCount(DateTime from, DateTime to);
-    public Task<MultiSeriesDataset[]> ByUniqueDocumentReferenceCount(DateTime from, DateTime to);
-    public Task<SingeSeriesDataset> UniqueDocumentReferenceByMovementCount(DateTime from, DateTime to);
+    public Task<MultiSeriesDatetimeDataset> ByCreated(DateTime from, DateTime to, AggregationPeriod aggregateBy = AggregationPeriod.Day);
+    public Task<SingleSeriesDataset> ByStatus(DateTime from, DateTime to);
+    public Task<MultiSeriesDataset> ByItemCount(DateTime from, DateTime to);
+    public Task<MultiSeriesDataset> ByUniqueDocumentReferenceCount(DateTime from, DateTime to);
+    public Task<SingleSeriesDataset> UniqueDocumentReferenceByMovementCount(DateTime from, DateTime to);
+    public Task<MultiSeriesDataset> ByCheck(DateTime from, DateTime to);
+    public Task<EntityDataset<AuditHistory>> GetHistory(string movementId);
 }
diff --git a/Btms.Analytics/ImportNotificationMetrics.cs b/Btms.Analytics/ImportNotificationMetrics.cs
@@ -39,14 +39,14 @@ public async Task RecordCurrentState()
     {
         var metrics = await _importService.ByCreated(DateTime.Today, DateTime.Now.NextHour());
 
-        foreach (var dataset in metrics)
+        foreach (var series in metrics.Series)
         {
-            var key = $"{AnalyticsMetricNames.MetricPrefix}.import-notifications.{dataset.MetricsKey()}.count";
+            var key = $"{AnalyticsMetricNames.MetricPrefix}.import-notifications.{series.MetricsKey()}.count";
             if (_metrics.TryGetValue(key, out var instrument))
             {
                 if (instrument is Gauge<int> g)
                 {
-                    g.Record(dataset.Periods[0].Value);
+                    g.Record(series.Periods[0].Value);
                 }
                 else
                 {

diff --git a/Btms.Analytics/ImportNotificationsAggregationService.cs b/Btms.Analytics/ImportNotificationsAggregationService.cs
@@ -12,7 +12,7 @@ namespace Btms.Analytics;
 
 public class ImportNotificationsAggregationService(IMongoDbContext context, ILogger<ImportNotificationsAggregationService> logger) : IImportNotificationsAggregationService
 {
-    public Task<MultiSeriesDatetimeDataset[]> ByCreated(DateTime from, DateTime to, AggregationPeriod aggregateBy = AggregationPeriod.Day)
+    public Task<MultiSeriesDatetimeDataset> ByCreated(DateTime from, DateTime to, AggregationPeriod aggregateBy = AggregationPeriod.Day)
     {
         var dateRange = AnalyticsHelpers.CreateDateRange(from, to, aggregateBy);
 
@@ -25,7 +25,7 @@ string CreateDatasetName(BsonDocument b) =>
         return Aggregate(dateRange, CreateDatasetName, matchFilter, "$createdSource", aggregateBy);
     }
 
-    public Task<MultiSeriesDatetimeDataset[]> ByArrival(DateTime from, DateTime to, AggregationPeriod aggregateBy = AggregationPeriod.Day)
+    public Task<MultiSeriesDatetimeDataset> ByArrival(DateTime from, DateTime to, AggregationPeriod aggregateBy = AggregationPeriod.Day)
     {
         var dateRange = AnalyticsHelpers.CreateDateRange(from, to, aggregateBy);
 
@@ -38,7 +38,7 @@ string CreateDatasetName(BsonDocument b) =>
         return Aggregate(dateRange, CreateDatasetName, matchFilter, "$partOne.arrivesAt", aggregateBy);
     }
 
-    public Task<SingeSeriesDataset> ByStatus(DateTime from, DateTime to)
+    public Task<SingleSeriesDataset> ByStatus(DateTime from, DateTime to)
     {
         var data = context
             .Notifications
@@ -48,13 +48,13 @@ public Task<SingeSeriesDataset> ByStatus(DateTime from, DateTime to)
             .ToDictionary(g => AnalyticsHelpers.GetLinkedName(g.Linked, g.ImportNotificationType.AsString()),
                 g => g.Count);
 
-        return Task.FromResult(new SingeSeriesDataset
+        return Task.FromResult(new SingleSeriesDataset
         {
             Values = AnalyticsHelpers.GetImportNotificationSegments().ToDictionary(title => title, title => data.GetValueOrDefault(title, 0))
         });
     }
 
-    public Task<MultiSeriesDataset[]> ByCommodityCount(DateTime from, DateTime to)
+    public Task<MultiSeriesDataset> ByCommodityCount(DateTime from, DateTime to)
     {
         var query = context
             .Notifications
@@ -72,8 +72,6 @@ public Task<MultiSeriesDataset[]> ByCommodityCount(DateTime from, DateTime to)
             .GroupBy(r => new { r.Key.ImportNotificationType, r.Key.Linked })
             .ToList();
 
-        // var maxCommodities = result.Max(r => r.Max(i => i.Key.CommodityCount));
-
         var maxCommodities = result.Count > 0 ?
             result.Max(r => r.Any() ? r.Max(i => i.Key.CommodityCount) : 0) : 0;
 
@@ -94,24 +92,23 @@ public Task<MultiSeriesDataset[]> ByCommodityCount(DateTime from, DateTime to)
                 g => g.NotificationCount);
 
 
-        return Task.FromResult(
-            AnalyticsHelpers.GetImportNotificationSegments()
-                .Select(title => new MultiSeriesDataset(title, "ItemCount")
+        return Task.FromResult(new MultiSeriesDataset()
+        {
+            Series = AnalyticsHelpers.GetImportNotificationSegments()
+                .Select(title => new Series(title, "ItemCount")
                 {
-                    // Results = asDictionary.AsResultList(title, maxCommodities)
                     Results = Enumerable.Range(0, maxCommodities)
                         .Select(i => new ByNumericDimensionResult
                         {
                             Dimension = i,
                             Value = asDictionary.GetValueOrDefault(new { Title=title, CommodityCount = i })
                         }).ToList()
-                }
-                )
-                .AsOrderedArray(d => d.Name)
-            );
+                })
+                .ToList()
+        });
     }
 
-    private Task<MultiSeriesDatetimeDataset[]> Aggregate(DateTime[] dateRange, Func<BsonDocument, string> createDatasetName, Expression<Func<ImportNotification, bool>> filter, string dateField, AggregationPeriod aggregateBy)
+    private Task<MultiSeriesDatetimeDataset> Aggregate(DateTime[] dateRange, Func<BsonDocument, string> createDatasetName, Expression<Func<ImportNotification, bool>> filter, string dateField, AggregationPeriod aggregateBy)
     {
         var truncateBy = aggregateBy == AggregationPeriod.Hour ? "hour" : "day";
 
@@ -133,6 +130,6 @@ private Task<MultiSeriesDatetimeDataset[]> Aggregate(DateTime[] dateRange, Func<
 
         logger.LogDebug("Aggregated Data {Result}", output.ToList().ToJsonString());
 
-        return Task.FromResult(output);
+        return Task.FromResult(new MultiSeriesDatetimeDataset() { Series = output.ToList() });
     }
 }