From 36ff1636fe727f1922d88a9eeefb6b8b64ee8bfe Mon Sep 17 00:00:00 2001 From: Craig Edmunds Date: Thu, 19 Dec 2024 10:23:00 +0000 Subject: [PATCH] CDMS-200 analytics adjustments for novembers prod data --- .../Extensions/AnalyticsExtensions.cs | 6 ++--- .../IImportNotificationsAggregationService.cs | 1 + .../ImportNotificationsAggregationService.cs | 27 ++++++++++++++++--- Btms.Backend/Config/AnalyticsDashboards.cs | 4 +++ Btms.Model/Movement.cs | 6 ++++- .../Relationships/RelationshipDataItem.cs | 4 +-- 6 files changed, 38 insertions(+), 10 deletions(-) diff --git a/Btms.Analytics/Extensions/AnalyticsExtensions.cs b/Btms.Analytics/Extensions/AnalyticsExtensions.cs index 76ed099..d63dbbd 100644 --- a/Btms.Analytics/Extensions/AnalyticsExtensions.cs +++ b/Btms.Analytics/Extensions/AnalyticsExtensions.cs @@ -132,10 +132,10 @@ internal static IEnumerable Execute( private static void LogExecutedMongoString(this ILogger logger, IQueryable source) { var stages = ((IMongoQueryProvider)source.Provider).LoggedStages; - - logger.LogInformation("[{Query}]", string.Join(",", stages.Select(s => s.ToString()).ToArray())); + var query = string.Join(",", stages.Select(s => s.ToString()).ToArray()); + logger.LogInformation("[{Query}]", query); } - + public static async Task AsIDataset(this Task ms) { await ms; diff --git a/Btms.Analytics/IImportNotificationsAggregationService.cs b/Btms.Analytics/IImportNotificationsAggregationService.cs index a6b4063..51faac4 100644 --- a/Btms.Analytics/IImportNotificationsAggregationService.cs +++ b/Btms.Analytics/IImportNotificationsAggregationService.cs @@ -6,4 +6,5 @@ public interface IImportNotificationsAggregationService public Task ByArrival(DateTime from, DateTime to, AggregationPeriod aggregateBy = AggregationPeriod.Day); public Task ByStatus(DateTime from, DateTime to); public Task ByCommodityCount(DateTime from, DateTime to); + public Task ByVersionCount(DateTime from, DateTime to, AggregationPeriod aggregateBy = AggregationPeriod.Day); } \ No newline at end of file diff --git a/Btms.Analytics/ImportNotificationsAggregationService.cs b/Btms.Analytics/ImportNotificationsAggregationService.cs index 26ed8f3..425b424 100644 --- a/Btms.Analytics/ImportNotificationsAggregationService.cs +++ b/Btms.Analytics/ImportNotificationsAggregationService.cs @@ -22,7 +22,7 @@ public Task ByCreated(DateTime from, DateTime to, Ag string CreateDatasetName(BsonDocument b) => AnalyticsHelpers.GetLinkedName(b["_id"]["linked"].ToBoolean(), b["_id"]["importNotificationType"].ToString()!.FromImportNotificationTypeEnumString()); - return Aggregate(dateRange, CreateDatasetName, matchFilter, "$createdSource", aggregateBy); + return AggregateByLinkedAndNotificationType(dateRange, CreateDatasetName, matchFilter, "$createdSource", aggregateBy); } public Task ByArrival(DateTime from, DateTime to, AggregationPeriod aggregateBy = AggregationPeriod.Day) @@ -35,7 +35,7 @@ public Task ByArrival(DateTime from, DateTime to, Ag string CreateDatasetName(BsonDocument b) => AnalyticsHelpers.GetLinkedName(b["_id"]["linked"].ToBoolean(), b["_id"]["importNotificationType"].ToString()!.FromImportNotificationTypeEnumString()); - return Aggregate(dateRange, CreateDatasetName, matchFilter, "$partOne.arrivesAt", aggregateBy); + return AggregateByLinkedAndNotificationType(dateRange, CreateDatasetName, matchFilter, "$partOne.arrivesAt", aggregateBy); } public Task ByStatus(DateTime from, DateTime to) @@ -59,11 +59,17 @@ public Task ByCommodityCount(DateTime from, DateTime to) var query = context .Notifications .Where(n => n.CreatedSource >= from && n.CreatedSource < to) + // .Select(n => new + // { + // ImportNotificationType = n.ImportNotificationType, + // Relationships = n.Relationships, + // Commodities = n.Commodities + // }) .GroupBy(n => new { ImportNotificationType = n.ImportNotificationType!.Value, Linked = n.Relationships.Movements.Data.Count > 0, - CommodityCount = n.Commodities.Count() + CommodityCount = n.Commodities == null ? 0 : n.Commodities.Count() }) .Select(g => new { g.Key, Count = g.Count() }); @@ -108,7 +114,20 @@ public Task ByCommodityCount(DateTime from, DateTime to) }); } - private Task Aggregate(DateTime[] dateRange, Func createDatasetName, Expression> filter, string dateField, AggregationPeriod aggregateBy) + public Task ByVersionCount(DateTime from, DateTime to, AggregationPeriod aggregateBy = AggregationPeriod.Day) + { + var dateRange = AnalyticsHelpers.CreateDateRange(from, to, aggregateBy); + + Expression> matchFilter = n => + n.CreatedSource >= from && n.CreatedSource < to; + + string CreateDatasetName(BsonDocument b) => + AnalyticsHelpers.GetLinkedName(b["_id"]["linked"].ToBoolean(), b["_id"]["importNotificationType"].ToString()!.FromImportNotificationTypeEnumString()); + + return AggregateByLinkedAndNotificationType(dateRange, CreateDatasetName, matchFilter, "$createdSource", aggregateBy); + } + + private Task AggregateByLinkedAndNotificationType(DateTime[] dateRange, Func createDatasetName, Expression> filter, string dateField, AggregationPeriod aggregateBy) { var truncateBy = aggregateBy == AggregationPeriod.Hour ? "hour" : "day"; diff --git a/Btms.Backend/Config/AnalyticsDashboards.cs b/Btms.Backend/Config/AnalyticsDashboards.cs index ef4c1c0..2c8244e 100644 --- a/Btms.Backend/Config/AnalyticsDashboards.cs +++ b/Btms.Backend/Config/AnalyticsDashboards.cs @@ -73,6 +73,10 @@ public static async Task> GetCharts( { "lastMonthDecisionsByStatus", () => movementsService.ByDecision(DateTime.Today.MonthAgo(), DateTime.Now).AsIDataset() + }, + { + "importNotificationVersionsByVersionCount", + () => importService.ByVersionCount(DateTime.Today.AddMonths(-3), DateTime.Today).AsIDataset() } }; diff --git a/Btms.Model/Movement.cs b/Btms.Model/Movement.cs index 0b70088..80f555d 100644 --- a/Btms.Model/Movement.cs +++ b/Btms.Model/Movement.cs @@ -137,10 +137,14 @@ public bool MergeDecision(string path, AlvsClearanceRequest clearanceRequest) } var decisionAuditContext = new Dictionary>(); - decisionAuditContext.Add("movements", new Dictionary() + decisionAuditContext.Add("clearanceRequests", new Dictionary() { { clearanceRequest.Header!.EntryReference!, clearanceRequest.Header!.EntryVersionNumber!.ToString()! } }); + decisionAuditContext.Add("decisions", new Dictionary() + { + { clearanceRequest.Header!.EntryReference!, clearanceRequest.Header!.DecisionNumber!.ToString()! } + }); decisionAuditContext.Add("importNotifications", new Dictionary() { { "todo", "todo" } diff --git a/Btms.Model/Relationships/RelationshipDataItem.cs b/Btms.Model/Relationships/RelationshipDataItem.cs index 9a66878..b8b2f68 100644 --- a/Btms.Model/Relationships/RelationshipDataItem.cs +++ b/Btms.Model/Relationships/RelationshipDataItem.cs @@ -61,7 +61,7 @@ public static RelationshipDataItem CreateFromNotification(ImportNotification not SourceItem = movement.Items .Find(x => x.Documents!.ToList().Exists(d => d.DocumentReference!.Contains(matchReference))) ?.ItemNumber, - DestinationItem = notification.Commodities.FirstOrDefault()?.ComplementId, + DestinationItem = notification.Commodities!.FirstOrDefault()?.ComplementId, Links = new ResourceLink { Self = LinksBuilder.Notification.BuildSelfNotificationLink(notification.Id!) }, MatchingLevel = 1 }; @@ -74,7 +74,7 @@ public static RelationshipDataItem CreateFromMovement(ImportNotification notific Matched = matched, Type = "movements", Id = movement.Id!, - SourceItem = notification.Commodities.FirstOrDefault()?.ComplementId, + SourceItem = notification.Commodities!.FirstOrDefault()?.ComplementId, DestinationItem = movement.Items .Find(x => x.Documents!.ToList().Exists(d => d.DocumentReference!.Contains(matchReference))) ?.ItemNumber,