CDMS-200 analytics adjustments for novembers prod data

DEFRA · Dec 19, 2024 · 36ff163 · 36ff163
1 parent 5862145
commit 36ff163
Show file tree

Hide file tree

Showing 6 changed files with 38 additions and 10 deletions.
diff --git a/Btms.Analytics/Extensions/AnalyticsExtensions.cs b/Btms.Analytics/Extensions/AnalyticsExtensions.cs
@@ -132,10 +132,10 @@ internal static IEnumerable<TSource> Execute<TSource>(
     private static void LogExecutedMongoString(this ILogger logger, IQueryable source)
     {
         var stages = ((IMongoQueryProvider)source.Provider).LoggedStages;
-
-        logger.LogInformation("[{Query}]", string.Join(",", stages.Select(s => s.ToString()).ToArray()));
+        var query = string.Join(",", stages.Select(s => s.ToString()).ToArray());
+        logger.LogInformation("[{Query}]", query);
     }
-
+    
     public static async Task<IDataset> AsIDataset(this Task<MultiSeriesDatetimeDataset> ms)
     {
         await ms;

diff --git a/Btms.Analytics/IImportNotificationsAggregationService.cs b/Btms.Analytics/IImportNotificationsAggregationService.cs
@@ -6,4 +6,5 @@ public interface IImportNotificationsAggregationService
     public Task<MultiSeriesDatetimeDataset> ByArrival(DateTime from, DateTime to, AggregationPeriod aggregateBy = AggregationPeriod.Day);
     public Task<SingleSeriesDataset> ByStatus(DateTime from, DateTime to);
     public Task<MultiSeriesDataset> ByCommodityCount(DateTime from, DateTime to);
+    public Task<MultiSeriesDatetimeDataset> ByVersionCount(DateTime from, DateTime to, AggregationPeriod aggregateBy = AggregationPeriod.Day);
 }
diff --git a/Btms.Analytics/ImportNotificationsAggregationService.cs b/Btms.Analytics/ImportNotificationsAggregationService.cs
@@ -22,7 +22,7 @@ public Task<MultiSeriesDatetimeDataset> ByCreated(DateTime from, DateTime to, Ag
         string CreateDatasetName(BsonDocument b) =>
             AnalyticsHelpers.GetLinkedName(b["_id"]["linked"].ToBoolean(), b["_id"]["importNotificationType"].ToString()!.FromImportNotificationTypeEnumString());
 
-        return Aggregate(dateRange, CreateDatasetName, matchFilter, "$createdSource", aggregateBy);
+        return AggregateByLinkedAndNotificationType(dateRange, CreateDatasetName, matchFilter, "$createdSource", aggregateBy);
     }
 
     public Task<MultiSeriesDatetimeDataset> ByArrival(DateTime from, DateTime to, AggregationPeriod aggregateBy = AggregationPeriod.Day)
@@ -35,7 +35,7 @@ public Task<MultiSeriesDatetimeDataset> ByArrival(DateTime from, DateTime to, Ag
         string CreateDatasetName(BsonDocument b) =>
             AnalyticsHelpers.GetLinkedName(b["_id"]["linked"].ToBoolean(), b["_id"]["importNotificationType"].ToString()!.FromImportNotificationTypeEnumString());
 
-        return Aggregate(dateRange, CreateDatasetName, matchFilter, "$partOne.arrivesAt", aggregateBy);
+        return AggregateByLinkedAndNotificationType(dateRange, CreateDatasetName, matchFilter, "$partOne.arrivesAt", aggregateBy);
     }
 
     public Task<SingleSeriesDataset> ByStatus(DateTime from, DateTime to)
@@ -59,11 +59,17 @@ public Task<MultiSeriesDataset> ByCommodityCount(DateTime from, DateTime to)
         var query = context
             .Notifications
             .Where(n => n.CreatedSource >= from && n.CreatedSource < to)
+            // .Select(n => new
+            // {
+            //     ImportNotificationType = n.ImportNotificationType,
+            //     Relationships = n.Relationships,
+            //     Commodities = n.Commodities
+            // })
             .GroupBy(n => new
             {
                 ImportNotificationType = n.ImportNotificationType!.Value,
                 Linked = n.Relationships.Movements.Data.Count > 0,
-                CommodityCount = n.Commodities.Count()
+                CommodityCount = n.Commodities == null ? 0 : n.Commodities.Count()
             })
             .Select(g => new { g.Key, Count = g.Count() });
 
@@ -108,7 +114,20 @@ public Task<MultiSeriesDataset> ByCommodityCount(DateTime from, DateTime to)
         });
     }
 
-    private Task<MultiSeriesDatetimeDataset> Aggregate(DateTime[] dateRange, Func<BsonDocument, string> createDatasetName, Expression<Func<ImportNotification, bool>> filter, string dateField, AggregationPeriod aggregateBy)
+    public Task<MultiSeriesDatetimeDataset> ByVersionCount(DateTime from, DateTime to, AggregationPeriod aggregateBy = AggregationPeriod.Day)
+    {
+        var dateRange = AnalyticsHelpers.CreateDateRange(from, to, aggregateBy);
+
+        Expression<Func<ImportNotification, bool>> matchFilter = n =>
+            n.CreatedSource >= from && n.CreatedSource < to;
+
+        string CreateDatasetName(BsonDocument b) =>
+            AnalyticsHelpers.GetLinkedName(b["_id"]["linked"].ToBoolean(), b["_id"]["importNotificationType"].ToString()!.FromImportNotificationTypeEnumString());
+
+        return AggregateByLinkedAndNotificationType(dateRange, CreateDatasetName, matchFilter, "$createdSource", aggregateBy);
+    }
+
+    private Task<MultiSeriesDatetimeDataset> AggregateByLinkedAndNotificationType(DateTime[] dateRange, Func<BsonDocument, string> createDatasetName, Expression<Func<ImportNotification, bool>> filter, string dateField, AggregationPeriod aggregateBy)
     {
         var truncateBy = aggregateBy == AggregationPeriod.Hour ? "hour" : "day";
 

diff --git a/Btms.Backend/Config/AnalyticsDashboards.cs b/Btms.Backend/Config/AnalyticsDashboards.cs
@@ -73,6 +73,10 @@ public static async Task<IDictionary<string, IDataset>> GetCharts(
             {
                 "lastMonthDecisionsByStatus",
                 () => movementsService.ByDecision(DateTime.Today.MonthAgo(), DateTime.Now).AsIDataset()
+            },
+            {
+                "importNotificationVersionsByVersionCount",
+                () => importService.ByVersionCount(DateTime.Today.AddMonths(-3), DateTime.Today).AsIDataset()
             }
         };
 

diff --git a/Btms.Model/Movement.cs b/Btms.Model/Movement.cs
@@ -137,10 +137,14 @@ public bool MergeDecision(string path, AlvsClearanceRequest clearanceRequest)
         }
 
         var decisionAuditContext = new Dictionary<string, Dictionary<string, string>>();
-        decisionAuditContext.Add("movements", new Dictionary<string, string>()
+        decisionAuditContext.Add("clearanceRequests", new Dictionary<string, string>()
         {
             { clearanceRequest.Header!.EntryReference!, clearanceRequest.Header!.EntryVersionNumber!.ToString()! }
         });
+        decisionAuditContext.Add("decisions", new Dictionary<string, string>()
+        {
+            { clearanceRequest.Header!.EntryReference!, clearanceRequest.Header!.DecisionNumber!.ToString()! }
+        });
         decisionAuditContext.Add("importNotifications", new Dictionary<string, string>()
         {
             { "todo", "todo" }

diff --git a/Btms.Model/Relationships/RelationshipDataItem.cs b/Btms.Model/Relationships/RelationshipDataItem.cs
@@ -61,7 +61,7 @@ public static RelationshipDataItem CreateFromNotification(ImportNotification not
             SourceItem = movement.Items
                 .Find(x => x.Documents!.ToList().Exists(d => d.DocumentReference!.Contains(matchReference)))
                 ?.ItemNumber,
-            DestinationItem = notification.Commodities.FirstOrDefault()?.ComplementId,
+            DestinationItem = notification.Commodities!.FirstOrDefault()?.ComplementId,
             Links = new ResourceLink { Self = LinksBuilder.Notification.BuildSelfNotificationLink(notification.Id!) },
             MatchingLevel = 1
         };
@@ -74,7 +74,7 @@ public static RelationshipDataItem CreateFromMovement(ImportNotification notific
             Matched = matched,
             Type = "movements",
             Id = movement.Id!,
-            SourceItem = notification.Commodities.FirstOrDefault()?.ComplementId,
+            SourceItem = notification.Commodities!.FirstOrDefault()?.ComplementId,
             DestinationItem = movement.Items
                 .Find(x => x.Documents!.ToList().Exists(d => d.DocumentReference!.Contains(matchReference)))
                 ?.ItemNumber,