diff --git a/Btms.Analytics/DatasetDimensions.cs b/Btms.Analytics/DatasetDimensions.cs index 6a0d4ca4..1bf9d458 100644 --- a/Btms.Analytics/DatasetDimensions.cs +++ b/Btms.Analytics/DatasetDimensions.cs @@ -51,10 +51,11 @@ public class StringBucketDimensionResult : IDimensionResult public int Value { get; set; } } -public class AuditHistory(AuditEntry auditEntry, string resourceType, string resourceId) +public class AuditHistory(AuditEntry auditEntry, string resourceType, string resourceApiPrefix, string resourceId) { public AuditEntry AuditEntry { get; set; } = auditEntry; public string ResourceType { get; set; } = resourceType; + public string ResourceApiPrefix { get; set; } = resourceApiPrefix; public string ResourceId { get; set; } = resourceId; } diff --git a/Btms.Analytics/MovementExceptions.cs b/Btms.Analytics/MovementExceptions.cs index b0668023..bdcde7d2 100644 --- a/Btms.Analytics/MovementExceptions.cs +++ b/Btms.Analytics/MovementExceptions.cs @@ -20,6 +20,8 @@ public class MovementExceptions(IMongoDbContext context, ILogger logger) .Where(m => country == null || m.DispatchCountryCode == country ) .Select(m => new { + // TODO - we should think about pre-calculating this stuff and storing it on the movement... + Id = m.Id, UpdatedSource = m.UpdatedSource, Updated = m.Updated, @@ -28,7 +30,14 @@ public class MovementExceptions(IMongoDbContext context, ILogger logger) LinkedCheds = m.Relationships.Notifications.Data.Count, ItemCount = m.Items.Count, HasMatchDecisions = m.AlvsDecisions.Any(d => d.Context.AlvsAnyMatch), - HasNotificationRelationships = m.Relationships.Notifications.Data.Count > 0 + DecisionMatched = !m.AlvsDecisions + .OrderBy(d => d.Context.AlvsDecisionNumber) + .Reverse() + .First() + .Context.DecisionMatched, + HasNotificationRelationships = m.Relationships.Notifications.Data.Count > 0, + ContiguousAlvsClearanceRequestVersionsFrom1 = + m.ClearanceRequests.Select(c => c.Header!.EntryVersionNumber) }) .Select(m => new { @@ -43,7 +52,9 @@ public class MovementExceptions(IMongoDbContext context, ILogger logger) HasNotificationRelationships = m.HasNotificationRelationships, Total = m.MaxDecisionNumber + m.MaxEntryVersion + m.LinkedCheds + m.ItemCount, // TODO - can we include CHED versions here too? - TotalDocumentVersions = m.MaxDecisionNumber + m.MaxEntryVersion + m.LinkedCheds + TotalDocumentVersions = m.MaxDecisionNumber + m.MaxEntryVersion + m.LinkedCheds, + DecisionMatched = m.DecisionMatched, + ContiguousAlvsClearanceRequestVersionsFrom1 = m.ContiguousAlvsClearanceRequestVersionsFrom1.Count() == m.MaxEntryVersion }); var moreComplexMovementsQuery = simplifiedMovementView @@ -51,7 +62,7 @@ public class MovementExceptions(IMongoDbContext context, ILogger logger) if (summary) { - exceptionsSummary.Values.Add("complexMovement", moreComplexMovementsQuery.Count()); + exceptionsSummary.Values.Add("Complex Movement", moreComplexMovementsQuery.Count()); } else { @@ -80,12 +91,43 @@ public class MovementExceptions(IMongoDbContext context, ILogger logger) if (summary) { - exceptionsSummary.Values.Add("alvsLinksButNotBtms", movementsWhereAlvsLinksButNotBtmsQuery.Count()); + exceptionsSummary.Values.Add("Alvs has match decisions but no Btms links", movementsWhereAlvsLinksButNotBtmsQuery.Count()); } else { exceptionsResult.AddRange(movementsWhereAlvsLinksButNotBtmsQuery - .OrderBy(a => -a.Total) + .OrderBy(a => a.Total) + .Take(10) + .Execute(logger) + .Select(r => + new ExceptionResult() + { + Resource = "Movement", + Id = r.Id!, + UpdatedSource = r.UpdatedSource!.Value, + Updated = r.Updated, + ItemCount = r.ItemCount, + MaxEntryVersion = r.MaxEntryVersion, + MaxDecisionNumber = r.MaxDecisionNumber, + LinkedCheds = r.LinkedCheds, + Reason = "Alvs has match decisions but no Btms links" + }) + ); + } + + var movementsWhereWeHaveAndContigousVersionsButDecisionsAreDifferentQuery = simplifiedMovementView + .Where(r => + r.ContiguousAlvsClearanceRequestVersionsFrom1 && r.DecisionMatched + && r.HasNotificationRelationships); + + if (summary) + { + exceptionsSummary.Values.Add("BTMS Links But Decision Wrong", movementsWhereWeHaveAndContigousVersionsButDecisionsAreDifferentQuery.Count()); + } + else + { + exceptionsResult.AddRange(movementsWhereWeHaveAndContigousVersionsButDecisionsAreDifferentQuery + .OrderBy(a => a.Total) .Take(10) .Execute(logger) .Select(r => @@ -99,7 +141,7 @@ public class MovementExceptions(IMongoDbContext context, ILogger logger) MaxEntryVersion = r.MaxEntryVersion, MaxDecisionNumber = r.MaxDecisionNumber, LinkedCheds = r.LinkedCheds, - Reason = "Alvs has match decisions but we don't have links" + Reason = "BTMS Links But Decision Wrong" }) ); } diff --git a/Btms.Analytics/MovementsAggregationService.cs b/Btms.Analytics/MovementsAggregationService.cs index ff9f8b4b..3cb201b1 100644 --- a/Btms.Analytics/MovementsAggregationService.cs +++ b/Btms.Analytics/MovementsAggregationService.cs @@ -173,12 +173,12 @@ public Task UniqueDocumentReferenceByMovementCount(DateTime .Where(n => notificationIds.Contains(n.Id)) .SelectMany(n => n.AuditEntries .Select(a => - new AuditHistory(a, $"ImportNotification", n.Id!) + new AuditHistory(a, "ImportNotification", "import-notifications", n.Id!) ) ); var entries = movement!.AuditEntries - .Select(a => new AuditHistory(a, "Movement", movementId)) + .Select(a => new AuditHistory(a, "Movement", "movements", movementId)) .Concat(notificationEntries); entries = entries.OrderBy(a => a.AuditEntry.CreatedSource); @@ -329,165 +329,8 @@ public Task> .ToList() }; - return Task.FromResult(r); - - // return DefaultSummarisedBucketResult(); + return Task.FromResult(r); } - - // public Task> ByDecisionComplex(DateTime from, DateTime to) - // { - // var mongoQuery = context - // .Movements - // // .Aggregate() - // .Where(m => m.CreatedSource >= from && m.CreatedSource < to) - // .Select(m => new - // { - // MovementInfo = new - // { - // Id = m.Id, - // UpdatedSource = m.UpdatedSource, - // Updated = m.Updated, - // Movement = m - // }, - // // ReSharper disable once ConditionIsAlwaysTrueOrFalseAccordingToNullableAPIContract - // // Get the most recent decision record from both systems - // AlvsDecision = (m.Decisions == null - // ? null - // : m.Decisions - // .Where(d => d.ServiceHeader!.SourceSystem == "ALVS") - // .OrderBy(d => d.ServiceHeader!.ServiceCalled) - // .Reverse() - // .FirstOrDefault()) - // // Creates a default item & check so we don't lose - // // it in the selectmany below - // ?? new CdsClearanceRequest() - // { - // Items = new [] - // { - // new Items() - // { - // Checks = new [] - // { - // new Check() - // { - // CheckCode = "XXX", - // DecisionCode = "XXX" - // } - // } - // } - // } - // }, - // BtmsDecision = m.Decisions == null - // ? null - // : m.Decisions - // .Where(d => d.ServiceHeader!.SourceSystem == "BTMS") - // .OrderBy(d => d.ServiceHeader!.ServiceCalled) - // .Reverse() - // .FirstOrDefault() - // }) - // .SelectMany(m => - // m.AlvsDecision!.Items! - // .SelectMany(i => - // (i.Checks - // ?? new[] { new Check() { CheckCode = "XXX", DecisionCode = "XXX" } }) - // .Select(c => - // new - // { - // m.MovementInfo, - // AlvsDecisionInfo = c.CheckCode == "XXX" ? null : new - // { - // Decision = m.AlvsDecision, - // DecisionNumber = m.AlvsDecision!.Header!.DecisionNumber, - // EntryVersion = m.AlvsDecision!.Header!.EntryVersionNumber, - // ItemNumber = i.ItemNumber, - // CheckCode = c.CheckCode, - // DecisionCode = c.DecisionCode, - // }, - // BtmsDecisionInfo = new - // { - // Decision = m.BtmsDecision, - // DecisionCode = m.BtmsDecision == null || m.BtmsDecision.Items == null - // ? null - // : m.BtmsDecision.Items! - // .First(bi => bi.ItemNumber == i.ItemNumber) - // .Checks! - // .First(ch => ch.CheckCode == c.CheckCode) - // .DecisionCode - // } - // } - // ) - // ) - // .Select(a => new - // { - // a.MovementInfo, - // a.AlvsDecisionInfo, - // a.BtmsDecisionInfo, - // Classification = - // a.BtmsDecisionInfo == null ? "Btms Decision Not Present" : - // a.AlvsDecisionInfo == null ? "Alvs Decision Not Present" : - // - // // TODO : we may want to try to consider clearance request version as well as the decision code - // a.BtmsDecisionInfo.DecisionCode == a.AlvsDecisionInfo.DecisionCode ? "Btms Made Same Decision As Alvs" : - // a.MovementInfo.Movement.Decisions - // .Any(d => d.Header!.DecisionNumber == 1) ? "Alvs Decision Version 1 Not Present" : - // a.MovementInfo.Movement.ClearanceRequests - // .Any(d => d.Header!.EntryVersionNumber == 1) ? "Alvs Clearance Request Version 1 Not Present" : - // a.AlvsDecisionInfo.DecisionNumber == 1 && a.AlvsDecisionInfo.EntryVersion == 1 ? "Single Entry And Decision Version" : - // a.BtmsDecisionInfo.DecisionCode != a.AlvsDecisionInfo.DecisionCode ? "Btms Made Different Decision To Alvs" : - // "Further Classification Needed" - // // "FurtherClassificationNeeded Check Code Is " + a.AlvsDecisionInfo.CheckCode - // }) - // ) - // - // // .Where(m => m.AlvsDecisionInfo == null) - // // .Where(m => m.AlvsDecision!.Items!.Any(i => i.Checks!.Any(c => c.CheckCode == "XXX"))) - // .GroupBy(check => new - // { - // check.Classification, - // check.AlvsDecisionInfo!.CheckCode, - // AlvsDecisionCode = check.AlvsDecisionInfo!.DecisionCode, - // BtmsDecisionCode=check.BtmsDecisionInfo!.DecisionCode - // }) - // .Select(g => new - // { - // g.Key, Count = g.Count() - // }) - // - // .Execute(logger); - // - // logger.LogDebug("Aggregated Data {Result}", mongoQuery.ToJsonString()); - // - // // Works - // var summary = new SingleSeriesDataset() { - // Values = mongoQuery - // .GroupBy(q => q.Key.Classification) - // .ToDictionary( - // g => g.Key, - // g => g.Sum(k => k.Count) - // ) - // }; - // - // var r = new SummarisedDataset() - // { - // Summary = summary, - // Result = mongoQuery.Select(a => new StringBucketDimensionResult() - // { - // Fields = new Dictionary() - // { - // { "Classification", a.Key.Classification }, - // { "CheckCode", a.Key.CheckCode! }, - // { "AlvsDecisionCode", a.Key.AlvsDecisionCode! }, - // { "BtmsDecisionCode", a.Key.BtmsDecisionCode! } - // }, - // Value = a.Count - // }) - // .OrderBy(r => r.Value) - // .Reverse() - // .ToList() - // }; - // - // return Task.FromResult(r); - // } private static Task> DefaultTabularDatasetByNameDimensionResult() { diff --git a/Btms.Backend/appsettings.Development.json b/Btms.Backend/appsettings.Development.json index 0ea6ac79..d075cf5d 100644 --- a/Btms.Backend/appsettings.Development.json +++ b/Btms.Backend/appsettings.Development.json @@ -43,7 +43,8 @@ "Credentials": { "PhaService": "eFrSxkkTurBU", "BtmsAdmin": "ebuzKbuuvdcm", - "DmpDeveloper": "AzELT2hz7sS5" + "DmpDeveloper": "AzELT2hz7sS5", + "BtmsFrontend": "a498HnJ79A5G" } } } diff --git a/Btms.Backend/appsettings.json b/Btms.Backend/appsettings.json index fd8ba5b6..09897890 100644 --- a/Btms.Backend/appsettings.json +++ b/Btms.Backend/appsettings.json @@ -31,9 +31,10 @@ }, "AuthKeyStore": { "Credentials": { - "PhaService": "", - "BtmsAdmin": "", - "DmpDeveloper": "" + "PhaService": "", + "BtmsAdmin": "", + "DmpDeveloper": "", + "BtmsFrontend": "" } } }