CDMS-200 includes dates & country filtering in movements analytics

Btms.Analytics/IImportNotificationsAggregationService.cs

@@ -6,5 +6,5 @@ public interface IImportNotificationsAggregationService
     public Task<MultiSeriesDatetimeDataset> ByArrival(DateTime from, DateTime to, AggregationPeriod aggregateBy = AggregationPeriod.Day);
     public Task<SingleSeriesDataset> ByStatus(DateTime from, DateTime to);
     public Task<MultiSeriesDataset> ByCommodityCount(DateTime from, DateTime to);
-    public Task<SingleSeriesDataset> ByMaxVersion(DateTime from, DateTime to);
+    public Task<SingleSeriesDataset> ByMaxVersion(DateTime from, DateTime to, string[]? chedTypes = null, string? country = null);
 }

Btms.Analytics/IMovementsAggregationService.cs

@@ -7,14 +7,15 @@ public interface IMovementsAggregationService
     public Task<MultiSeriesDatetimeDataset> ByCreated(DateTime from, DateTime to, AggregationPeriod aggregateBy = AggregationPeriod.Day);
     public Task<SingleSeriesDataset> ByStatus(DateTime from, DateTime to);
     public Task<MultiSeriesDataset> ByItemCount(DateTime from, DateTime to);
-    public Task<SummarisedDataset<SingleSeriesDataset, StringBucketDimensionResult>> ByDecision(DateTime from, DateTime to);
+    public Task<SummarisedDataset<SingleSeriesDataset, StringBucketDimensionResult>> ByDecision(DateTime from, DateTime to, string[]? chedTypes = null, string? country = null);
     // public Task<TabularDataset<ByNameDimensionResult>> ByDecisionAndLinkStatus(DateTime from, DateTime to);
     public Task<MultiSeriesDataset> ByUniqueDocumentReferenceCount(DateTime from, DateTime to);
     public Task<SingleSeriesDataset> UniqueDocumentReferenceByMovementCount(DateTime from, DateTime to);
-    public Task<MultiSeriesDataset> ByCheck(DateTime from, DateTime to);
+    // public Task<MultiSeriesDataset> ByCheck(DateTime from, DateTime to, string[]? chedTypes = null, string? country = null);
     public Task<EntityDataset<AuditHistory>?> GetHistory(string movementId);
-    public Task<SingleSeriesDataset> ByMaxVersion(DateTime from, DateTime to);
-    public Task<SingleSeriesDataset> ByMaxDecisionNumber(DateTime from, DateTime to);
-    public Task<List<ExceptionResult>> GetExceptions(DateTime from, DateTime to);
+    public Task<SingleSeriesDataset> ByMaxVersion(DateTime from, DateTime to, string[]? chedTypes = null, string? country = null);
+    public Task<SingleSeriesDataset> ByMaxDecisionNumber(DateTime from, DateTime to, string[]? chedTypes = null, string? country = null);
+    public Task<List<ExceptionResult>> GetExceptions(DateTime from, DateTime to, string[]? chedTypes = null, string? country = null);
+    public Task<SingleSeriesDataset> ExceptionSummary(DateTime from, DateTime to, string[]? chedTypes = null, string? country = null);
 
 }

Btms.Analytics/ImportNotificationsAggregationService.cs

@@ -115,11 +115,12 @@ public Task<MultiSeriesDataset> ByCommodityCount(DateTime from, DateTime to)
         });
     }
 
-    public Task<SingleSeriesDataset> ByMaxVersion(DateTime from, DateTime to)
+    public Task<SingleSeriesDataset> ByMaxVersion(DateTime from, DateTime to, string[]? chedTypes = null, string? country = null)
     {
         var data = context
             .Notifications
             .Where(n => n.CreatedSource >= from && n.CreatedSource < to)
+            .Where(m => country == null || m.PartOne!.Route!.TransitingStates!.Contains(country))
             .GroupBy(n => new { MaxVersion =
                 n.AuditEntries.Where(a => a.CreatedBy == "Ipaffs").Max(a => a.Version )
             })

Btms.Analytics/MovementExceptions.cs

@@ -0,0 +1,109 @@
+using Btms.Analytics.Extensions;
+using Btms.Backend.Data;
+using Microsoft.Extensions.Logging;
+
+namespace Btms.Analytics;
+
+public class MovementExceptions(IMongoDbContext context, ILogger logger)
+{
+    //Returns a summary of the exceptions or a list
+    // Means we can share the same anonymous / query code without needing to create loads
+    // of classes
+    public (SingleSeriesDataset summary, List<ExceptionResult>) GetAllExceptions(DateTime from, DateTime to, bool summary, string[]? chedTypes = null, string? country = null)
+    {
+        var exceptionsSummary = new SingleSeriesDataset();
+        var exceptionsResult = new List<ExceptionResult>();
+
+        var simplifiedMovementView = context
+            .Movements
+            .Where(n => n.CreatedSource >= from && n.CreatedSource < to)
+            .Where(m => country == null || m.DispatchCountryCode == country )
+            .Select(m => new
+            {
+                Id = m.Id,
+                UpdatedSource = m.UpdatedSource,
+                Updated = m.Updated,
+                MaxDecisionNumber = m.Decisions.Max(d => d.Header!.DecisionNumber) ?? 0,
+                MaxEntryVersion = m.ClearanceRequests.Max(c => c.Header!.EntryVersionNumber) ?? 0,
+                LinkedCheds = m.Relationships.Notifications.Data.Count,
+                ItemCount = m.Items.Count,
+                HasMatchDecisions = m.AlvsDecisions.Any(d => d.Context.AlvsAnyMatch),
+                HasNotificationRelationships = m.Relationships.Notifications.Data.Count > 0
+            })
+            .Select(m => new
+            {
+                Id = m.Id,
+                UpdatedSource = m.UpdatedSource,
+                Updated = m.Updated,
+                MaxDecisionNumber = m.MaxDecisionNumber,
+                MaxEntryVersion = m.MaxEntryVersion,
+                LinkedCheds = m.LinkedCheds,
+                ItemCount = m.ItemCount,
+                HasMatchDecisions = m.HasMatchDecisions,
+                HasNotificationRelationships = m.HasNotificationRelationships,
+                Total = m.MaxDecisionNumber + m.MaxEntryVersion + m.LinkedCheds + m.ItemCount,
+                // TODO - can we include CHED versions here too?
+                TotalDocumentVersions = m.MaxDecisionNumber + m.MaxEntryVersion + m.LinkedCheds
+            });
+
+        var moreComplexMovementsQuery = simplifiedMovementView
+            .Where(r => r.TotalDocumentVersions > 5);
+
+        if (summary)
+        {
+            exceptionsSummary.Values.Add("complexMovement", moreComplexMovementsQuery.Count());
+        }
+        else
+        {
+            exceptionsResult.AddRange(moreComplexMovementsQuery
+                .OrderBy(a => -a.Total)
+                .Take(10)
+                .Execute(logger)
+                .Select(r =>
+                    new ExceptionResult()
+                    {
+                        Resource = "Movement",
+                        Id = r.Id!,
+                        UpdatedSource = r.UpdatedSource!.Value,
+                        Updated = r.Updated,
+                        ItemCount = r.ItemCount,
+                        MaxEntryVersion = r.MaxEntryVersion,
+                        MaxDecisionNumber = r.MaxDecisionNumber,
+                        LinkedCheds = r.LinkedCheds,
+                        Reason = "High Number Of Messages"
+                    })
+            );
+        }
+
+        var movementsWhereAlvsLinksButNotBtmsQuery = simplifiedMovementView
+            .Where(r => r.HasMatchDecisions && !r.HasNotificationRelationships);
+
+        if (summary)
+        {
+            exceptionsSummary.Values.Add("alvsLinksButNotBtms", movementsWhereAlvsLinksButNotBtmsQuery.Count());
+        }
+        else
+        {
+            exceptionsResult.AddRange(movementsWhereAlvsLinksButNotBtmsQuery
+                .OrderBy(a => -a.Total)
+                .Take(10)
+                .Execute(logger)
+                .Select(r =>
+                    new ExceptionResult()
+                    {
+                        Resource = "Movement",
+                        Id = r.Id!,
+                        UpdatedSource = r.UpdatedSource!.Value,
+                        Updated = r.Updated,
+                        ItemCount = r.ItemCount,
+                        MaxEntryVersion = r.MaxEntryVersion,
+                        MaxDecisionNumber = r.MaxDecisionNumber,
+                        LinkedCheds = r.LinkedCheds,
+                        Reason = "Alvs has match decisions but we don't have links"
+                    })
+            );
+        }
+
+        return (exceptionsSummary, exceptionsResult);
+    }
+}