From 11754a52fa5225735c8cf21746f9b8f032b4d5c1 Mon Sep 17 00:00:00 2001 From: Craig Edmunds Date: Mon, 23 Dec 2024 21:07:33 +0000 Subject: [PATCH] CDMS-200 includes dates & country filtering in movements analytics --- .../IImportNotificationsAggregationService.cs | 2 +- .../IMovementsAggregationService.cs | 11 +- .../ImportNotificationsAggregationService.cs | 3 +- Btms.Analytics/MovementExceptions.cs | 109 +++++ Btms.Analytics/MovementsAggregationService.cs | 391 ++++++++---------- Btms.Backend.IntegrationTests/SmokeTests.cs | 2 +- Btms.Backend/Config/AnalyticsDashboards.cs | 22 +- Btms.Backend/Endpoints/AnalyticsEndpoints.cs | 8 +- Btms.Model/Auditing/AuditEntry.cs | 1 + Btms.Model/Auditing/IAuditContext.cs | 4 + Btms.Model/Cds/AlvsDecision.cs | 27 +- Btms.Model/Movement.cs | 8 +- TestDataGenerator/Helpers/DataHelpers.cs | 14 +- TestDataGenerator/Program.cs | 20 +- .../Properties/launchSettings.json | 19 +- .../ChedPSimpleMatchScenarioGenerator.cs | 9 +- 16 files changed, 392 insertions(+), 258 deletions(-) create mode 100644 Btms.Analytics/MovementExceptions.cs diff --git a/Btms.Analytics/IImportNotificationsAggregationService.cs b/Btms.Analytics/IImportNotificationsAggregationService.cs index 4ccdee3..49ea778 100644 --- a/Btms.Analytics/IImportNotificationsAggregationService.cs +++ b/Btms.Analytics/IImportNotificationsAggregationService.cs @@ -6,5 +6,5 @@ public interface IImportNotificationsAggregationService public Task ByArrival(DateTime from, DateTime to, AggregationPeriod aggregateBy = AggregationPeriod.Day); public Task ByStatus(DateTime from, DateTime to); public Task ByCommodityCount(DateTime from, DateTime to); - public Task ByMaxVersion(DateTime from, DateTime to); + public Task ByMaxVersion(DateTime from, DateTime to, string[]? chedTypes = null, string? country = null); } \ No newline at end of file diff --git a/Btms.Analytics/IMovementsAggregationService.cs b/Btms.Analytics/IMovementsAggregationService.cs index a13deba..968a08f 100644 --- a/Btms.Analytics/IMovementsAggregationService.cs +++ b/Btms.Analytics/IMovementsAggregationService.cs @@ -7,14 +7,15 @@ public interface IMovementsAggregationService public Task ByCreated(DateTime from, DateTime to, AggregationPeriod aggregateBy = AggregationPeriod.Day); public Task ByStatus(DateTime from, DateTime to); public Task ByItemCount(DateTime from, DateTime to); - public Task> ByDecision(DateTime from, DateTime to); + public Task> ByDecision(DateTime from, DateTime to, string[]? chedTypes = null, string? country = null); // public Task> ByDecisionAndLinkStatus(DateTime from, DateTime to); public Task ByUniqueDocumentReferenceCount(DateTime from, DateTime to); public Task UniqueDocumentReferenceByMovementCount(DateTime from, DateTime to); - public Task ByCheck(DateTime from, DateTime to); + // public Task ByCheck(DateTime from, DateTime to, string[]? chedTypes = null, string? country = null); public Task?> GetHistory(string movementId); - public Task ByMaxVersion(DateTime from, DateTime to); - public Task ByMaxDecisionNumber(DateTime from, DateTime to); - public Task> GetExceptions(DateTime from, DateTime to); + public Task ByMaxVersion(DateTime from, DateTime to, string[]? chedTypes = null, string? country = null); + public Task ByMaxDecisionNumber(DateTime from, DateTime to, string[]? chedTypes = null, string? country = null); + public Task> GetExceptions(DateTime from, DateTime to, string[]? chedTypes = null, string? country = null); + public Task ExceptionSummary(DateTime from, DateTime to, string[]? chedTypes = null, string? country = null); } \ No newline at end of file diff --git a/Btms.Analytics/ImportNotificationsAggregationService.cs b/Btms.Analytics/ImportNotificationsAggregationService.cs index f083694..8443d37 100644 --- a/Btms.Analytics/ImportNotificationsAggregationService.cs +++ b/Btms.Analytics/ImportNotificationsAggregationService.cs @@ -115,11 +115,12 @@ public Task ByCommodityCount(DateTime from, DateTime to) }); } - public Task ByMaxVersion(DateTime from, DateTime to) + public Task ByMaxVersion(DateTime from, DateTime to, string[]? chedTypes = null, string? country = null) { var data = context .Notifications .Where(n => n.CreatedSource >= from && n.CreatedSource < to) + .Where(m => country == null || m.PartOne!.Route!.TransitingStates!.Contains(country)) .GroupBy(n => new { MaxVersion = n.AuditEntries.Where(a => a.CreatedBy == "Ipaffs").Max(a => a.Version ) }) diff --git a/Btms.Analytics/MovementExceptions.cs b/Btms.Analytics/MovementExceptions.cs new file mode 100644 index 0000000..b066802 --- /dev/null +++ b/Btms.Analytics/MovementExceptions.cs @@ -0,0 +1,109 @@ +using Btms.Analytics.Extensions; +using Btms.Backend.Data; +using Microsoft.Extensions.Logging; + +namespace Btms.Analytics; + +public class MovementExceptions(IMongoDbContext context, ILogger logger) +{ + //Returns a summary of the exceptions or a list + // Means we can share the same anonymous / query code without needing to create loads + // of classes + public (SingleSeriesDataset summary, List) GetAllExceptions(DateTime from, DateTime to, bool summary, string[]? chedTypes = null, string? country = null) + { + var exceptionsSummary = new SingleSeriesDataset(); + var exceptionsResult = new List(); + + var simplifiedMovementView = context + .Movements + .Where(n => n.CreatedSource >= from && n.CreatedSource < to) + .Where(m => country == null || m.DispatchCountryCode == country ) + .Select(m => new + { + Id = m.Id, + UpdatedSource = m.UpdatedSource, + Updated = m.Updated, + MaxDecisionNumber = m.Decisions.Max(d => d.Header!.DecisionNumber) ?? 0, + MaxEntryVersion = m.ClearanceRequests.Max(c => c.Header!.EntryVersionNumber) ?? 0, + LinkedCheds = m.Relationships.Notifications.Data.Count, + ItemCount = m.Items.Count, + HasMatchDecisions = m.AlvsDecisions.Any(d => d.Context.AlvsAnyMatch), + HasNotificationRelationships = m.Relationships.Notifications.Data.Count > 0 + }) + .Select(m => new + { + Id = m.Id, + UpdatedSource = m.UpdatedSource, + Updated = m.Updated, + MaxDecisionNumber = m.MaxDecisionNumber, + MaxEntryVersion = m.MaxEntryVersion, + LinkedCheds = m.LinkedCheds, + ItemCount = m.ItemCount, + HasMatchDecisions = m.HasMatchDecisions, + HasNotificationRelationships = m.HasNotificationRelationships, + Total = m.MaxDecisionNumber + m.MaxEntryVersion + m.LinkedCheds + m.ItemCount, + // TODO - can we include CHED versions here too? + TotalDocumentVersions = m.MaxDecisionNumber + m.MaxEntryVersion + m.LinkedCheds + }); + + var moreComplexMovementsQuery = simplifiedMovementView + .Where(r => r.TotalDocumentVersions > 5); + + if (summary) + { + exceptionsSummary.Values.Add("complexMovement", moreComplexMovementsQuery.Count()); + } + else + { + exceptionsResult.AddRange(moreComplexMovementsQuery + .OrderBy(a => -a.Total) + .Take(10) + .Execute(logger) + .Select(r => + new ExceptionResult() + { + Resource = "Movement", + Id = r.Id!, + UpdatedSource = r.UpdatedSource!.Value, + Updated = r.Updated, + ItemCount = r.ItemCount, + MaxEntryVersion = r.MaxEntryVersion, + MaxDecisionNumber = r.MaxDecisionNumber, + LinkedCheds = r.LinkedCheds, + Reason = "High Number Of Messages" + }) + ); + } + + var movementsWhereAlvsLinksButNotBtmsQuery = simplifiedMovementView + .Where(r => r.HasMatchDecisions && !r.HasNotificationRelationships); + + if (summary) + { + exceptionsSummary.Values.Add("alvsLinksButNotBtms", movementsWhereAlvsLinksButNotBtmsQuery.Count()); + } + else + { + exceptionsResult.AddRange(movementsWhereAlvsLinksButNotBtmsQuery + .OrderBy(a => -a.Total) + .Take(10) + .Execute(logger) + .Select(r => + new ExceptionResult() + { + Resource = "Movement", + Id = r.Id!, + UpdatedSource = r.UpdatedSource!.Value, + Updated = r.Updated, + ItemCount = r.ItemCount, + MaxEntryVersion = r.MaxEntryVersion, + MaxDecisionNumber = r.MaxDecisionNumber, + LinkedCheds = r.LinkedCheds, + Reason = "Alvs has match decisions but we don't have links" + }) + ); + } + + return (exceptionsSummary, exceptionsResult); + } +} \ No newline at end of file diff --git a/Btms.Analytics/MovementsAggregationService.cs b/Btms.Analytics/MovementsAggregationService.cs index dd8f575..ff9f8b4 100644 --- a/Btms.Analytics/MovementsAggregationService.cs +++ b/Btms.Analytics/MovementsAggregationService.cs @@ -186,11 +186,12 @@ public Task UniqueDocumentReferenceByMovementCount(DateTime return new EntityDataset(entries); } - public Task ByMaxVersion(DateTime from, DateTime to) + public Task ByMaxVersion(DateTime from, DateTime to, string[]? chedTypes = null, string? country = null) { var data = context .Movements .Where(n => n.CreatedSource >= from && n.CreatedSource < to) + .Where(m => country == null || m.DispatchCountryCode == country ) .GroupBy(n => new { MaxVersion = n.ClearanceRequests.Max(a => a.Header!.EntryVersionNumber ) }) @@ -203,11 +204,12 @@ public Task ByMaxVersion(DateTime from, DateTime to) }); } - public Task ByMaxDecisionNumber(DateTime from, DateTime to) + public Task ByMaxDecisionNumber(DateTime from, DateTime to, string[]? chedTypes = null, string? country = null) { var data = context .Movements .Where(n => n.CreatedSource >= from && n.CreatedSource < to) + .Where(m => country == null || m.DispatchCountryCode == country ) .GroupBy(n => new { MaxVersion = n.Decisions.Max(a => a.Header!.DecisionNumber ) }) @@ -220,73 +222,28 @@ public Task ByMaxDecisionNumber(DateTime from, DateTime to) }); } - public Task> GetExceptions(DateTime from, DateTime to) + public Task> GetExceptions(DateTime from, DateTime to, string[]? chedTypes = null, string? country = null) { - var mongoQuery = context - .Movements - .Select(m => new - { - Id = m.Id, - UpdatedSource = m.UpdatedSource, - Updated = m.Updated, - MaxDecisionNumber = m.Decisions.Max(d => d.Header!.DecisionNumber) ?? 0, - MaxEntryVersion = m.ClearanceRequests.Max(c => c.Header!.EntryVersionNumber) ?? 0, - LinkedCheds = m.Relationships.Notifications.Data.Count, - ItemCount = m.Items.Count - - }) - .Select(m => new - { - Id = m.Id, - UpdatedSource = m.UpdatedSource, - Updated = m.Updated, - MaxDecisionNumber = m.MaxDecisionNumber, - MaxEntryVersion = m.MaxEntryVersion, - LinkedCheds = m.LinkedCheds, - ItemCount = m.ItemCount, - Total = m.MaxDecisionNumber + m.MaxEntryVersion + m.LinkedCheds + m.ItemCount - }) - .OrderBy(a => -a.Total) - .Take(10) - .Execute(logger); - - var result = mongoQuery.Select(r => - new ExceptionResult() - { - Resource = "Movement", - Id = r.Id!, - UpdatedSource = r.UpdatedSource!.Value, - Updated = r.Updated, - ItemCount = r.ItemCount, - MaxEntryVersion = r.MaxEntryVersion, - MaxDecisionNumber = r.MaxDecisionNumber, - LinkedCheds = r.LinkedCheds, - Reason = "High Number Of Messages" - }).ToList(); - // var result = new TabularDataset() - // { - // Rows = mongoQuery.Select(r => - // new TabularDimensionRow() - // { - // Key = r.Id!, Columns = - // [ - // new ByNameDimensionResult() { Name = "ItemCount", Value = r.ItemCount }, - // new ByNameDimensionResult() { Name = "MaxEntryVersion", Value = r.MaxEntryVersion }, - // new ByNameDimensionResult() { Name = "MaxDecisionNumber", Value = r.MaxDecisionNumber }, - // new ByNameDimensionResult() { Name = "LinkedCheds", Value = r.LinkedCheds } - // ] - // } - // ).ToList() - // }; + var movementExceptions = new MovementExceptions(context, logger); + var (_, result) = movementExceptions.GetAllExceptions(from, to, false, chedTypes, country); return Task.FromResult(result); } - - public Task ByCheck(DateTime from, DateTime to) + + public Task ExceptionSummary(DateTime from, DateTime to, string[]? chedTypes = null, string? country = null) { - return Task.FromResult(new MultiSeriesDataset() ); + var movementExceptions = new MovementExceptions(context, logger); + var (summary, _) = movementExceptions.GetAllExceptions(from, to, true, chedTypes, country); + + return Task.FromResult(summary); } + // TODO : remove + // public Task ByCheck(DateTime from, DateTime to, string[]? chedTypes = null, string? country = null) + // { + // return Task.FromResult(new MultiSeriesDataset() ); + // } + private Task Aggregate(DateTime[] dateRange, Func createDatasetName, Expression> filter, string dateField, AggregationPeriod aggregateBy) { var truncateBy = aggregateBy == AggregationPeriod.Hour ? "hour" : "day"; @@ -319,7 +276,7 @@ private Task Aggregate(DateTime[] dateRange, Func /// public Task> ByDecision(DateTime from, - DateTime to) + DateTime to, string[]? chedTypes = null, string? country = null) { var mongoQuery = context .Movements @@ -377,160 +334,160 @@ public Task> // return DefaultSummarisedBucketResult(); } - public Task> ByDecisionComplex(DateTime from, DateTime to) - { - var mongoQuery = context - .Movements - // .Aggregate() - .Where(m => m.CreatedSource >= from && m.CreatedSource < to) - .Select(m => new - { - MovementInfo = new - { - Id = m.Id, - UpdatedSource = m.UpdatedSource, - Updated = m.Updated, - Movement = m - }, - // ReSharper disable once ConditionIsAlwaysTrueOrFalseAccordingToNullableAPIContract - // Get the most recent decision record from both systems - AlvsDecision = (m.Decisions == null - ? null - : m.Decisions - .Where(d => d.ServiceHeader!.SourceSystem == "ALVS") - .OrderBy(d => d.ServiceHeader!.ServiceCalled) - .Reverse() - .FirstOrDefault()) - // Creates a default item & check so we don't lose - // it in the selectmany below - ?? new CdsClearanceRequest() - { - Items = new [] - { - new Items() - { - Checks = new [] - { - new Check() - { - CheckCode = "XXX", - DecisionCode = "XXX" - } - } - } - } - }, - BtmsDecision = m.Decisions == null - ? null - : m.Decisions - .Where(d => d.ServiceHeader!.SourceSystem == "BTMS") - .OrderBy(d => d.ServiceHeader!.ServiceCalled) - .Reverse() - .FirstOrDefault() - }) - .SelectMany(m => - m.AlvsDecision!.Items! - .SelectMany(i => - (i.Checks - ?? new[] { new Check() { CheckCode = "XXX", DecisionCode = "XXX" } }) - .Select(c => - new - { - m.MovementInfo, - AlvsDecisionInfo = c.CheckCode == "XXX" ? null : new - { - Decision = m.AlvsDecision, - DecisionNumber = m.AlvsDecision!.Header!.DecisionNumber, - EntryVersion = m.AlvsDecision!.Header!.EntryVersionNumber, - ItemNumber = i.ItemNumber, - CheckCode = c.CheckCode, - DecisionCode = c.DecisionCode, - }, - BtmsDecisionInfo = new - { - Decision = m.BtmsDecision, - DecisionCode = m.BtmsDecision == null || m.BtmsDecision.Items == null - ? null - : m.BtmsDecision.Items! - .First(bi => bi.ItemNumber == i.ItemNumber) - .Checks! - .First(ch => ch.CheckCode == c.CheckCode) - .DecisionCode - } - } - ) - ) - .Select(a => new - { - a.MovementInfo, - a.AlvsDecisionInfo, - a.BtmsDecisionInfo, - Classification = - a.BtmsDecisionInfo == null ? "Btms Decision Not Present" : - a.AlvsDecisionInfo == null ? "Alvs Decision Not Present" : - - // TODO : we may want to try to consider clearance request version as well as the decision code - a.BtmsDecisionInfo.DecisionCode == a.AlvsDecisionInfo.DecisionCode ? "Btms Made Same Decision As Alvs" : - a.MovementInfo.Movement.Decisions - .Any(d => d.Header!.DecisionNumber == 1) ? "Alvs Decision Version 1 Not Present" : - a.MovementInfo.Movement.ClearanceRequests - .Any(d => d.Header!.EntryVersionNumber == 1) ? "Alvs Clearance Request Version 1 Not Present" : - a.AlvsDecisionInfo.DecisionNumber == 1 && a.AlvsDecisionInfo.EntryVersion == 1 ? "Single Entry And Decision Version" : - a.BtmsDecisionInfo.DecisionCode != a.AlvsDecisionInfo.DecisionCode ? "Btms Made Different Decision To Alvs" : - "Further Classification Needed" - // "FurtherClassificationNeeded Check Code Is " + a.AlvsDecisionInfo.CheckCode - }) - ) - - // .Where(m => m.AlvsDecisionInfo == null) - // .Where(m => m.AlvsDecision!.Items!.Any(i => i.Checks!.Any(c => c.CheckCode == "XXX"))) - .GroupBy(check => new - { - check.Classification, - check.AlvsDecisionInfo!.CheckCode, - AlvsDecisionCode = check.AlvsDecisionInfo!.DecisionCode, - BtmsDecisionCode=check.BtmsDecisionInfo!.DecisionCode - }) - .Select(g => new - { - g.Key, Count = g.Count() - }) - - .Execute(logger); - - logger.LogDebug("Aggregated Data {Result}", mongoQuery.ToJsonString()); - - // Works - var summary = new SingleSeriesDataset() { - Values = mongoQuery - .GroupBy(q => q.Key.Classification) - .ToDictionary( - g => g.Key, - g => g.Sum(k => k.Count) - ) - }; - - var r = new SummarisedDataset() - { - Summary = summary, - Result = mongoQuery.Select(a => new StringBucketDimensionResult() - { - Fields = new Dictionary() - { - { "Classification", a.Key.Classification }, - { "CheckCode", a.Key.CheckCode! }, - { "AlvsDecisionCode", a.Key.AlvsDecisionCode! }, - { "BtmsDecisionCode", a.Key.BtmsDecisionCode! } - }, - Value = a.Count - }) - .OrderBy(r => r.Value) - .Reverse() - .ToList() - }; - - return Task.FromResult(r); - } + // public Task> ByDecisionComplex(DateTime from, DateTime to) + // { + // var mongoQuery = context + // .Movements + // // .Aggregate() + // .Where(m => m.CreatedSource >= from && m.CreatedSource < to) + // .Select(m => new + // { + // MovementInfo = new + // { + // Id = m.Id, + // UpdatedSource = m.UpdatedSource, + // Updated = m.Updated, + // Movement = m + // }, + // // ReSharper disable once ConditionIsAlwaysTrueOrFalseAccordingToNullableAPIContract + // // Get the most recent decision record from both systems + // AlvsDecision = (m.Decisions == null + // ? null + // : m.Decisions + // .Where(d => d.ServiceHeader!.SourceSystem == "ALVS") + // .OrderBy(d => d.ServiceHeader!.ServiceCalled) + // .Reverse() + // .FirstOrDefault()) + // // Creates a default item & check so we don't lose + // // it in the selectmany below + // ?? new CdsClearanceRequest() + // { + // Items = new [] + // { + // new Items() + // { + // Checks = new [] + // { + // new Check() + // { + // CheckCode = "XXX", + // DecisionCode = "XXX" + // } + // } + // } + // } + // }, + // BtmsDecision = m.Decisions == null + // ? null + // : m.Decisions + // .Where(d => d.ServiceHeader!.SourceSystem == "BTMS") + // .OrderBy(d => d.ServiceHeader!.ServiceCalled) + // .Reverse() + // .FirstOrDefault() + // }) + // .SelectMany(m => + // m.AlvsDecision!.Items! + // .SelectMany(i => + // (i.Checks + // ?? new[] { new Check() { CheckCode = "XXX", DecisionCode = "XXX" } }) + // .Select(c => + // new + // { + // m.MovementInfo, + // AlvsDecisionInfo = c.CheckCode == "XXX" ? null : new + // { + // Decision = m.AlvsDecision, + // DecisionNumber = m.AlvsDecision!.Header!.DecisionNumber, + // EntryVersion = m.AlvsDecision!.Header!.EntryVersionNumber, + // ItemNumber = i.ItemNumber, + // CheckCode = c.CheckCode, + // DecisionCode = c.DecisionCode, + // }, + // BtmsDecisionInfo = new + // { + // Decision = m.BtmsDecision, + // DecisionCode = m.BtmsDecision == null || m.BtmsDecision.Items == null + // ? null + // : m.BtmsDecision.Items! + // .First(bi => bi.ItemNumber == i.ItemNumber) + // .Checks! + // .First(ch => ch.CheckCode == c.CheckCode) + // .DecisionCode + // } + // } + // ) + // ) + // .Select(a => new + // { + // a.MovementInfo, + // a.AlvsDecisionInfo, + // a.BtmsDecisionInfo, + // Classification = + // a.BtmsDecisionInfo == null ? "Btms Decision Not Present" : + // a.AlvsDecisionInfo == null ? "Alvs Decision Not Present" : + // + // // TODO : we may want to try to consider clearance request version as well as the decision code + // a.BtmsDecisionInfo.DecisionCode == a.AlvsDecisionInfo.DecisionCode ? "Btms Made Same Decision As Alvs" : + // a.MovementInfo.Movement.Decisions + // .Any(d => d.Header!.DecisionNumber == 1) ? "Alvs Decision Version 1 Not Present" : + // a.MovementInfo.Movement.ClearanceRequests + // .Any(d => d.Header!.EntryVersionNumber == 1) ? "Alvs Clearance Request Version 1 Not Present" : + // a.AlvsDecisionInfo.DecisionNumber == 1 && a.AlvsDecisionInfo.EntryVersion == 1 ? "Single Entry And Decision Version" : + // a.BtmsDecisionInfo.DecisionCode != a.AlvsDecisionInfo.DecisionCode ? "Btms Made Different Decision To Alvs" : + // "Further Classification Needed" + // // "FurtherClassificationNeeded Check Code Is " + a.AlvsDecisionInfo.CheckCode + // }) + // ) + // + // // .Where(m => m.AlvsDecisionInfo == null) + // // .Where(m => m.AlvsDecision!.Items!.Any(i => i.Checks!.Any(c => c.CheckCode == "XXX"))) + // .GroupBy(check => new + // { + // check.Classification, + // check.AlvsDecisionInfo!.CheckCode, + // AlvsDecisionCode = check.AlvsDecisionInfo!.DecisionCode, + // BtmsDecisionCode=check.BtmsDecisionInfo!.DecisionCode + // }) + // .Select(g => new + // { + // g.Key, Count = g.Count() + // }) + // + // .Execute(logger); + // + // logger.LogDebug("Aggregated Data {Result}", mongoQuery.ToJsonString()); + // + // // Works + // var summary = new SingleSeriesDataset() { + // Values = mongoQuery + // .GroupBy(q => q.Key.Classification) + // .ToDictionary( + // g => g.Key, + // g => g.Sum(k => k.Count) + // ) + // }; + // + // var r = new SummarisedDataset() + // { + // Summary = summary, + // Result = mongoQuery.Select(a => new StringBucketDimensionResult() + // { + // Fields = new Dictionary() + // { + // { "Classification", a.Key.Classification }, + // { "CheckCode", a.Key.CheckCode! }, + // { "AlvsDecisionCode", a.Key.AlvsDecisionCode! }, + // { "BtmsDecisionCode", a.Key.BtmsDecisionCode! } + // }, + // Value = a.Count + // }) + // .OrderBy(r => r.Value) + // .Reverse() + // .ToList() + // }; + // + // return Task.FromResult(r); + // } private static Task> DefaultTabularDatasetByNameDimensionResult() { diff --git a/Btms.Backend.IntegrationTests/SmokeTests.cs b/Btms.Backend.IntegrationTests/SmokeTests.cs index 1f88d56..85f63cc 100644 --- a/Btms.Backend.IntegrationTests/SmokeTests.cs +++ b/Btms.Backend.IntegrationTests/SmokeTests.cs @@ -72,7 +72,7 @@ await MakeSyncNotificationsRequest(new SyncNotificationsCommand jsonClientResponse.Data.Count.Should().Be(5); } - [Fact(Skip="Movement was refactored")] + [Fact(Skip="Unclear how Checks was being set as it's not present in the clearance request text file :|")] public async Task SyncDecisions() { //Arrange diff --git a/Btms.Backend/Config/AnalyticsDashboards.cs b/Btms.Backend/Config/AnalyticsDashboards.cs index eb447ab..69cd7cf 100644 --- a/Btms.Backend/Config/AnalyticsDashboards.cs +++ b/Btms.Backend/Config/AnalyticsDashboards.cs @@ -14,7 +14,7 @@ public static async Task> GetCharts( IMovementsAggregationService movementsService, string[] chartsToRender, string[] chedTypes, - string? countryOfOrigin, + string? country, DateTime? dateFrom, DateTime? dateTo ) @@ -76,20 +76,24 @@ public static async Task> GetCharts( () => importService.ByCommodityCount(DateTime.Today.MonthAgo(), DateTime.Now).AsIDataset() }, { - "lastMonthsDecisionsByDecisionCode", - () => movementsService.ByDecision(dateFrom ?? DateTime.Today.MonthAgo(), dateTo ?? DateTime.Now).AsIDataset() + "decisionsByDecisionCode", + () => movementsService.ByDecision(dateFrom ?? DateTime.Today.MonthAgo(), dateTo ?? DateTime.Now, chedTypes, country).AsIDataset() }, { - "allImportNotificationsByVersion", - () => importService.ByMaxVersion(DateTime.Today.AddMonths(-3), DateTime.Today).AsIDataset() + "importNotificationsByVersion", + () => importService.ByMaxVersion(dateFrom ?? DateTime.Today.AddMonths(-3), dateTo ?? DateTime.Today, chedTypes, country).AsIDataset() }, { - "allMovementsByMaxEntryVersion", - () => movementsService.ByMaxVersion(DateTime.Today.AddMonths(-3), DateTime.Today).AsIDataset() + "movementsByMaxEntryVersion", + () => movementsService.ByMaxVersion(dateFrom ?? DateTime.Today.AddMonths(-3), dateTo ?? DateTime.Today, chedTypes, country).AsIDataset() }, { - "allMovementsByMaxDecisionNumber", - () => movementsService.ByMaxDecisionNumber(DateTime.Today.AddMonths(-3), DateTime.Today).AsIDataset() + "movementsByMaxDecisionNumber", + () => movementsService.ByMaxDecisionNumber(dateFrom ?? DateTime.Today.AddMonths(-3), dateTo ?? DateTime.Today, chedTypes, country).AsIDataset() + }, + { + "movementsExceptions", + () => movementsService.ExceptionSummary(dateFrom ?? DateTime.Today.AddMonths(-3), dateTo ?? DateTime.Today, chedTypes, country).AsIDataset() } }; diff --git a/Btms.Backend/Endpoints/AnalyticsEndpoints.cs b/Btms.Backend/Endpoints/AnalyticsEndpoints.cs index f01d668..402709a 100644 --- a/Btms.Backend/Endpoints/AnalyticsEndpoints.cs +++ b/Btms.Backend/Endpoints/AnalyticsEndpoints.cs @@ -34,10 +34,14 @@ private static async Task Timeline( return Results.NotFound(); } private static async Task Exceptions( - [FromServices] IMovementsAggregationService movementsService) + [FromServices] IMovementsAggregationService movementsService, + [FromQuery(Name = "chedType")] string[] chedTypes, + [FromQuery(Name = "country")] string? country, + [FromQuery(Name = "dateFrom")] DateTime? dateFrom, + [FromQuery(Name = "dateTo")] DateTime? dateTo) { var result - = await movementsService.GetExceptions(DateTime.MinValue, DateTime.Today); + = await movementsService.GetExceptions(dateFrom ?? DateTime.MinValue, dateTo ?? DateTime.Today); if (result.HasValue()) { diff --git a/Btms.Model/Auditing/AuditEntry.cs b/Btms.Model/Auditing/AuditEntry.cs index 06f73f2..25ee0df 100644 --- a/Btms.Model/Auditing/AuditEntry.cs +++ b/Btms.Model/Auditing/AuditEntry.cs @@ -4,6 +4,7 @@ using Btms.Model.ChangeLog; using Btms.Model.Extensions; using Json.Patch; +using MongoDB.Bson.Serialization.Attributes; namespace Btms.Model.Auditing; diff --git a/Btms.Model/Auditing/IAuditContext.cs b/Btms.Model/Auditing/IAuditContext.cs index 3d231da..67d2cb3 100644 --- a/Btms.Model/Auditing/IAuditContext.cs +++ b/Btms.Model/Auditing/IAuditContext.cs @@ -1,5 +1,9 @@ +using Btms.Model.Cds; +using MongoDB.Bson.Serialization.Attributes; + namespace Btms.Model.Auditing; +// [BsonKnownTypes(typeof(DecisionContext)] public interface IAuditContext { diff --git a/Btms.Model/Cds/AlvsDecision.cs b/Btms.Model/Cds/AlvsDecision.cs index f58d3ab..1d48ec7 100644 --- a/Btms.Model/Cds/AlvsDecision.cs +++ b/Btms.Model/Cds/AlvsDecision.cs @@ -12,13 +12,11 @@ using System.Text.Json.Serialization; using System.Dynamic; using Btms.Model.Auditing; +using MongoDB.Bson.Serialization.Attributes; namespace Btms.Model.Cds; -/// -/// -/// public partial class ItemCheck { [Attr] @@ -38,10 +36,6 @@ public partial class ItemCheck public string? BtmsDecisionCode { get; set; } } - -/// -/// -/// public partial class DecisionContext : IAuditContext // { [Attr] @@ -68,6 +62,14 @@ public partial class DecisionContext : IAuditContext // [System.ComponentModel.Description("")] public bool DecisionMatched { get; set; } = default; + [Attr] + [System.ComponentModel.Description("")] + public bool BtmsAllMatch { get; set; } = default; + + [Attr] + [System.ComponentModel.Description("")] + public bool BtmsAnyMatch { get; set; } = default; + [Attr] [System.ComponentModel.Description("")] public bool BtmsAllNoMatch { get; set; } = default; @@ -100,6 +102,14 @@ public partial class DecisionContext : IAuditContext // [System.ComponentModel.Description("")] public bool BtmsAnyRelease { get; set; } = default; + [Attr] + [System.ComponentModel.Description("")] + public bool AlvsAllMatch { get; set; } = default; + + [Attr] + [System.ComponentModel.Description("")] + public bool AlvsAnyMatch { get; set; } = default; + [Attr] [System.ComponentModel.Description("")] public bool AlvsAllNoMatch { get; set; } = default; @@ -134,9 +144,6 @@ public partial class DecisionContext : IAuditContext // } -/// -/// -/// public partial class AlvsDecision // { [Attr] diff --git a/Btms.Model/Movement.cs b/Btms.Model/Movement.cs index eb2255b..dceb9b6 100644 --- a/Btms.Model/Movement.cs +++ b/Btms.Model/Movement.cs @@ -29,7 +29,7 @@ public class Movement : IMongoIdentifiable, IDataEntity, IAuditable [Attr] public List Decisions { get; set; } = default!; [Attr] public List AlvsDecisions { get; set; } = new List(); - + [Attr] public List Items { get; set; } = []; [Attr] @@ -216,6 +216,8 @@ private AlvsDecision FindBtmsPairAndUpdate(CdsClearanceRequest clearanceRequest) private void CompareDecisions(AlvsDecision alvsDecision, CdsClearanceRequest btmsDecision) { + alvsDecision.Context.AlvsAllMatch = alvsDecision.Checks.All(c => !c.AlvsDecisionCode.StartsWith('X')); + alvsDecision.Context.AlvsAnyMatch = alvsDecision.Checks.Any(c => !c.AlvsDecisionCode.StartsWith('X')); alvsDecision.Context.AlvsAllNoMatch = alvsDecision.Checks.All(c => c.AlvsDecisionCode.StartsWith('X')); alvsDecision.Context.AlvsAnyNoMatch = alvsDecision.Checks.Any(c => c.AlvsDecisionCode.StartsWith('X')); alvsDecision.Context.AlvsAllRefuse = alvsDecision.Checks.All(c => c.AlvsDecisionCode.StartsWith('N')); @@ -225,6 +227,10 @@ private void CompareDecisions(AlvsDecision alvsDecision, CdsClearanceRequest btm alvsDecision.Context.AlvsAllHold = alvsDecision.Checks.All(c => c.AlvsDecisionCode.StartsWith('H')); alvsDecision.Context.AlvsAnyHold = alvsDecision.Checks.Any(c => c.AlvsDecisionCode.StartsWith('H')); + alvsDecision.Context.BtmsAllMatch = alvsDecision.Checks.All( + c => c.BtmsDecisionCode != null && !c.BtmsDecisionCode.StartsWith('X')); + alvsDecision.Context.BtmsAnyMatch = alvsDecision.Checks.Any( + c => c.BtmsDecisionCode != null && !c.BtmsDecisionCode.StartsWith('X')); alvsDecision.Context.BtmsAllNoMatch = alvsDecision.Checks.All( c => c.BtmsDecisionCode != null && c.BtmsDecisionCode.StartsWith('X')); alvsDecision.Context.BtmsAnyNoMatch = alvsDecision.Checks.Any( diff --git a/TestDataGenerator/Helpers/DataHelpers.cs b/TestDataGenerator/Helpers/DataHelpers.cs index 92f1963..d4faf2f 100644 --- a/TestDataGenerator/Helpers/DataHelpers.cs +++ b/TestDataGenerator/Helpers/DataHelpers.cs @@ -3,6 +3,7 @@ using Btms.Types.Ipaffs.V1.Extensions; using Btms.Types.Alvs; using Btms.Types.Ipaffs; +using Decision = Btms.Types.Alvs.Decision; namespace TestDataGenerator.Helpers; @@ -18,6 +19,8 @@ internal static string BlobPath(this object resource, string rootPath) return n.BlobPath(rootPath); case AlvsClearanceRequest cr: return cr.BlobPath(rootPath); + case Decision d: + return d.BlobPath(rootPath); default: throw new InvalidDataException($"Unexpected type {resource.GetType().Name}"); } @@ -38,12 +41,21 @@ internal static string DateRef(this DateTime created) internal static string BlobPath(this AlvsClearanceRequest clearanceRequest, string rootPath) { var dateString = clearanceRequest.ServiceHeader!.ServiceCallTimestamp!.Value.ToString("yyyy/MM/dd"); - var subPath = clearanceRequest.Header!.DecisionNumber.HasValue() ? "DECISIONS" : "ALVS"; + var subPath = "ALVS"; return $"{rootPath}/{subPath}/{dateString}/{clearanceRequest.Header!.EntryReference!.Replace(".", "")}-{Guid.NewGuid()}.json"; } + internal static string BlobPath(this Decision decision, string rootPath) + { + var dateString = decision.ServiceHeader!.ServiceCallTimestamp!.Value.ToString("yyyy/MM/dd"); + var subPath = "DECISIONS"; + + return + $"{rootPath}/{subPath}/{dateString}/{decision.Header!.EntryReference!.Replace(".", "")}-{Guid.NewGuid()}.json"; + } + internal static string AsCdsEntryReference(this MatchIdentifier identifier) { return $"23GB9999{identifier.Identifier}"; diff --git a/TestDataGenerator/Program.cs b/TestDataGenerator/Program.cs index aa0e27c..378499d 100644 --- a/TestDataGenerator/Program.cs +++ b/TestDataGenerator/Program.cs @@ -64,8 +64,8 @@ private static async Task Main(string[] args) }, new { - Dataset = "LoadTest-One", - RootPath = "GENERATED-LOADTEST-ONE", + Dataset = "One", + RootPath = "GENERATED-ONE", Scenarios = new[] { app.CreateScenarioConfig(1, 1), @@ -74,8 +74,8 @@ private static async Task Main(string[] args) }, new { - Dataset = "LoadTest-Basic", - RootPath = "GENERATED-LOADTEST-BASIC", + Dataset = "Basic", + RootPath = "GENERATED-BASIC", Scenarios = new[] { app.CreateScenarioConfig(3, 7), @@ -119,6 +119,18 @@ private static async Task Main(string[] args) } }, new + { + Dataset = "90Dx1", + RootPath = "GENERATED-90Dx1", + Scenarios = + new[] + { + app.CreateScenarioConfig(1, 90), + app.CreateScenarioConfig(1, 90), + app.CreateScenarioConfig(1, 90) + } + }, + new { Dataset = "PHA", RootPath = "GENERATED-PHA", diff --git a/TestDataGenerator/Properties/launchSettings.json b/TestDataGenerator/Properties/launchSettings.json index b44e8bf..a01300c 100644 --- a/TestDataGenerator/Properties/launchSettings.json +++ b/TestDataGenerator/Properties/launchSettings.json @@ -12,9 +12,9 @@ "AZURE_TENANT_ID": "c9d74090-b4e6-4b04-981d-e6757a160812" } }, - "Generate LoadTest-One": { + "Generate One": { "commandName": "Project", - "commandLineArgs": "LoadTest-One", + "commandLineArgs": "One", "environmentVariables": { "DMP_ENVIRONMENT": "dev", "DMP_SERVICE_BUS_NAME": "DEVTREINFSB1001", @@ -23,9 +23,9 @@ "AZURE_TENANT_ID": "c9d74090-b4e6-4b04-981d-e6757a160812" } }, - "Generate LoadTest-Basic": { + "Generate Basic": { "commandName": "Project", - "commandLineArgs": "LoadTest-Basic", + "commandLineArgs": "Basic", "environmentVariables": { "DMP_ENVIRONMENT": "dev", "DMP_SERVICE_BUS_NAME": "DEVTREINFSB1001", @@ -56,6 +56,17 @@ "AZURE_TENANT_ID": "c9d74090-b4e6-4b04-981d-e6757a160812" } }, + "Generate 90Dx1": { + "commandName": "Project", + "commandLineArgs": "90Dx1", + "environmentVariables": { + "DMP_ENVIRONMENT": "dev", + "DMP_SERVICE_BUS_NAME": "DEVTREINFSB1001", + "DMP_BLOB_STORAGE_NAME": "devdmpinfdl1001", + "DMP_SLOT": "1003", + "AZURE_TENANT_ID": "c9d74090-b4e6-4b04-981d-e6757a160812" + } + }, "Generate LoadTest-Condensed": { "commandName": "Project", "commandLineArgs": "LoadTest-Condensed", diff --git a/TestDataGenerator/Scenarios/ChedPSimpleMatchScenarioGenerator.cs b/TestDataGenerator/Scenarios/ChedPSimpleMatchScenarioGenerator.cs index 1e650f2..e39d7b8 100644 --- a/TestDataGenerator/Scenarios/ChedPSimpleMatchScenarioGenerator.cs +++ b/TestDataGenerator/Scenarios/ChedPSimpleMatchScenarioGenerator.cs @@ -17,12 +17,17 @@ public override GeneratorResult Generate(int scenario, int item, DateTime entryD logger.LogInformation("Created {NotificationReferenceNumber}", notification.ReferenceNumber); + // TODO - check with Matt what a sensible checkCode, decision & other fields we need to + // implement a 'real world' test here + var checkCode = "H2019"; + var decisionCode = "H01"; + var clearanceRequest = GetClearanceRequestBuilder("cr-one-item") .WithCreationDate(entryDate.AddHours(2), false) .WithArrivalDateTimeOffset(notification.PartOne!.ArrivalDate, notification.PartOne!.ArrivalTime) .WithReferenceNumber(notification.ReferenceNumber!) .WithEntryVersionNumber(1) - .WithItem("N853", "16041421", "Tuna ROW CHEDP", 900) + .WithItem("N853", "16041421", "Tuna ROW CHEDP", 900, checkCode) .ValidateAndBuild(); logger.LogInformation("Created {EntryReference}", clearanceRequest.Header!.EntryReference); @@ -31,7 +36,7 @@ public override GeneratorResult Generate(int scenario, int item, DateTime entryD .WithCreationDate(clearanceRequest.ServiceHeader!.ServiceCallTimestamp!.Value.AddHours(1), false) .WithReferenceNumber(notification.ReferenceNumber!) .WithEntryVersionNumber(1) - .WithItemAndCheck(1, "H222", "H01") + .WithItemAndCheck(1, checkCode, decisionCode) .ValidateAndBuild(); logger.LogInformation("Created {EntryReference}", alvsDecision.Header!.EntryReference);