Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

CDMS-200 additional exception for linked but wrong decision #29

Merged
merged 1 commit into from
Dec 24, 2024
Merged

CDMS-200 additional exception for linked but wrong decision #29

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion Btms.Analytics/DatasetDimensions.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,10 +51,11 @@ public class StringBucketDimensionResult : IDimensionResult
public int Value { get; set; }
}

public class AuditHistory(AuditEntry auditEntry, string resourceType, string resourceId)
public class AuditHistory(AuditEntry auditEntry, string resourceType, string resourceApiPrefix, string resourceId)
{
public AuditEntry AuditEntry { get; set; } = auditEntry;
public string ResourceType { get; set; } = resourceType;
public string ResourceApiPrefix { get; set; } = resourceApiPrefix;
public string ResourceId { get; set; } = resourceId;
}

Expand Down
54 changes: 48 additions & 6 deletions Btms.Analytics/MovementExceptions.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@ public class MovementExceptions(IMongoDbContext context, ILogger logger)
.Where(m => country == null || m.DispatchCountryCode == country )
.Select(m => new
{
// TODO - we should think about pre-calculating this stuff and storing it on the movement...

Id = m.Id,
UpdatedSource = m.UpdatedSource,
Updated = m.Updated,
Expand All @@ -28,7 +30,14 @@ public class MovementExceptions(IMongoDbContext context, ILogger logger)
LinkedCheds = m.Relationships.Notifications.Data.Count,
ItemCount = m.Items.Count,
HasMatchDecisions = m.AlvsDecisions.Any(d => d.Context.AlvsAnyMatch),
HasNotificationRelationships = m.Relationships.Notifications.Data.Count > 0
DecisionMatched = !m.AlvsDecisions
.OrderBy(d => d.Context.AlvsDecisionNumber)
.Reverse()
.First()
.Context.DecisionMatched,
HasNotificationRelationships = m.Relationships.Notifications.Data.Count > 0,
ContiguousAlvsClearanceRequestVersionsFrom1 =
m.ClearanceRequests.Select(c => c.Header!.EntryVersionNumber)
})
.Select(m => new
{
Expand All @@ -43,15 +52,17 @@ public class MovementExceptions(IMongoDbContext context, ILogger logger)
HasNotificationRelationships = m.HasNotificationRelationships,
Total = m.MaxDecisionNumber + m.MaxEntryVersion + m.LinkedCheds + m.ItemCount,
// TODO - can we include CHED versions here too?
TotalDocumentVersions = m.MaxDecisionNumber + m.MaxEntryVersion + m.LinkedCheds
TotalDocumentVersions = m.MaxDecisionNumber + m.MaxEntryVersion + m.LinkedCheds,
DecisionMatched = m.DecisionMatched,
ContiguousAlvsClearanceRequestVersionsFrom1 = m.ContiguousAlvsClearanceRequestVersionsFrom1.Count() == m.MaxEntryVersion
});

var moreComplexMovementsQuery = simplifiedMovementView
.Where(r => r.TotalDocumentVersions > 5);

if (summary)
{
exceptionsSummary.Values.Add("complexMovement", moreComplexMovementsQuery.Count());
exceptionsSummary.Values.Add("Complex Movement", moreComplexMovementsQuery.Count());
}
else
{
Expand Down Expand Up @@ -80,12 +91,43 @@ public class MovementExceptions(IMongoDbContext context, ILogger logger)

if (summary)
{
exceptionsSummary.Values.Add("alvsLinksButNotBtms", movementsWhereAlvsLinksButNotBtmsQuery.Count());
exceptionsSummary.Values.Add("Alvs has match decisions but no Btms links", movementsWhereAlvsLinksButNotBtmsQuery.Count());
}
else
{
exceptionsResult.AddRange(movementsWhereAlvsLinksButNotBtmsQuery
.OrderBy(a => -a.Total)
.OrderBy(a => a.Total)
.Take(10)
.Execute(logger)
.Select(r =>
new ExceptionResult()
{
Resource = "Movement",
Id = r.Id!,
UpdatedSource = r.UpdatedSource!.Value,
Updated = r.Updated,
ItemCount = r.ItemCount,
MaxEntryVersion = r.MaxEntryVersion,
MaxDecisionNumber = r.MaxDecisionNumber,
LinkedCheds = r.LinkedCheds,
Reason = "Alvs has match decisions but no Btms links"
})
);
}

var movementsWhereWeHaveAndContigousVersionsButDecisionsAreDifferentQuery = simplifiedMovementView
.Where(r =>
r.ContiguousAlvsClearanceRequestVersionsFrom1 && r.DecisionMatched
&& r.HasNotificationRelationships);

if (summary)
{
exceptionsSummary.Values.Add("BTMS Links But Decision Wrong", movementsWhereWeHaveAndContigousVersionsButDecisionsAreDifferentQuery.Count());
}
else
{
exceptionsResult.AddRange(movementsWhereWeHaveAndContigousVersionsButDecisionsAreDifferentQuery
.OrderBy(a => a.Total)
.Take(10)
.Execute(logger)
.Select(r =>
Expand All @@ -99,7 +141,7 @@ public class MovementExceptions(IMongoDbContext context, ILogger logger)
MaxEntryVersion = r.MaxEntryVersion,
MaxDecisionNumber = r.MaxDecisionNumber,
LinkedCheds = r.LinkedCheds,
Reason = "Alvs has match decisions but we don't have links"
Reason = "BTMS Links But Decision Wrong"
})
);
}
Expand Down
163 changes: 3 additions & 160 deletions Btms.Analytics/MovementsAggregationService.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -173,12 +173,12 @@ public Task<SingleSeriesDataset> UniqueDocumentReferenceByMovementCount(DateTime
.Where(n => notificationIds.Contains(n.Id))
.SelectMany(n => n.AuditEntries
.Select(a =>
new AuditHistory(a, $"ImportNotification", n.Id!)
new AuditHistory(a, "ImportNotification", "import-notifications", n.Id!)
)
);

var entries = movement!.AuditEntries
.Select(a => new AuditHistory(a, "Movement", movementId))
.Select(a => new AuditHistory(a, "Movement", "movements", movementId))
.Concat(notificationEntries);

entries = entries.OrderBy(a => a.AuditEntry.CreatedSource);
Expand Down Expand Up @@ -329,165 +329,8 @@ public Task<SummarisedDataset<SingleSeriesDataset, StringBucketDimensionResult>>
.ToList()
};

return Task.FromResult(r);

// return DefaultSummarisedBucketResult();
return Task.FromResult(r);
}

// public Task<SummarisedDataset<SingleSeriesDataset, StringBucketDimensionResult>> ByDecisionComplex(DateTime from, DateTime to)
// {
// var mongoQuery = context
// .Movements
// // .Aggregate()
// .Where(m => m.CreatedSource >= from && m.CreatedSource < to)
// .Select(m => new
// {
// MovementInfo = new
// {
// Id = m.Id,
// UpdatedSource = m.UpdatedSource,
// Updated = m.Updated,
// Movement = m
// },
// // ReSharper disable once ConditionIsAlwaysTrueOrFalseAccordingToNullableAPIContract
// // Get the most recent decision record from both systems
// AlvsDecision = (m.Decisions == null
// ? null
// : m.Decisions
// .Where(d => d.ServiceHeader!.SourceSystem == "ALVS")
// .OrderBy(d => d.ServiceHeader!.ServiceCalled)
// .Reverse()
// .FirstOrDefault())
// // Creates a default item & check so we don't lose
// // it in the selectmany below
// ?? new CdsClearanceRequest()
// {
// Items = new []
// {
// new Items()
// {
// Checks = new []
// {
// new Check()
// {
// CheckCode = "XXX",
// DecisionCode = "XXX"
// }
// }
// }
// }
// },
// BtmsDecision = m.Decisions == null
// ? null
// : m.Decisions
// .Where(d => d.ServiceHeader!.SourceSystem == "BTMS")
// .OrderBy(d => d.ServiceHeader!.ServiceCalled)
// .Reverse()
// .FirstOrDefault()
// })
// .SelectMany(m =>
// m.AlvsDecision!.Items!
// .SelectMany(i =>
// (i.Checks
// ?? new[] { new Check() { CheckCode = "XXX", DecisionCode = "XXX" } })
// .Select(c =>
// new
// {
// m.MovementInfo,
// AlvsDecisionInfo = c.CheckCode == "XXX" ? null : new
// {
// Decision = m.AlvsDecision,
// DecisionNumber = m.AlvsDecision!.Header!.DecisionNumber,
// EntryVersion = m.AlvsDecision!.Header!.EntryVersionNumber,
// ItemNumber = i.ItemNumber,
// CheckCode = c.CheckCode,
// DecisionCode = c.DecisionCode,
// },
// BtmsDecisionInfo = new
// {
// Decision = m.BtmsDecision,
// DecisionCode = m.BtmsDecision == null || m.BtmsDecision.Items == null
// ? null
// : m.BtmsDecision.Items!
// .First(bi => bi.ItemNumber == i.ItemNumber)
// .Checks!
// .First(ch => ch.CheckCode == c.CheckCode)
// .DecisionCode
// }
// }
// )
// )
// .Select(a => new
// {
// a.MovementInfo,
// a.AlvsDecisionInfo,
// a.BtmsDecisionInfo,
// Classification =
// a.BtmsDecisionInfo == null ? "Btms Decision Not Present" :
// a.AlvsDecisionInfo == null ? "Alvs Decision Not Present" :
//
// // TODO : we may want to try to consider clearance request version as well as the decision code
// a.BtmsDecisionInfo.DecisionCode == a.AlvsDecisionInfo.DecisionCode ? "Btms Made Same Decision As Alvs" :
// a.MovementInfo.Movement.Decisions
// .Any(d => d.Header!.DecisionNumber == 1) ? "Alvs Decision Version 1 Not Present" :
// a.MovementInfo.Movement.ClearanceRequests
// .Any(d => d.Header!.EntryVersionNumber == 1) ? "Alvs Clearance Request Version 1 Not Present" :
// a.AlvsDecisionInfo.DecisionNumber == 1 && a.AlvsDecisionInfo.EntryVersion == 1 ? "Single Entry And Decision Version" :
// a.BtmsDecisionInfo.DecisionCode != a.AlvsDecisionInfo.DecisionCode ? "Btms Made Different Decision To Alvs" :
// "Further Classification Needed"
// // "FurtherClassificationNeeded Check Code Is " + a.AlvsDecisionInfo.CheckCode
// })
// )
//
// // .Where(m => m.AlvsDecisionInfo == null)
// // .Where(m => m.AlvsDecision!.Items!.Any(i => i.Checks!.Any(c => c.CheckCode == "XXX")))
// .GroupBy(check => new
// {
// check.Classification,
// check.AlvsDecisionInfo!.CheckCode,
// AlvsDecisionCode = check.AlvsDecisionInfo!.DecisionCode,
// BtmsDecisionCode=check.BtmsDecisionInfo!.DecisionCode
// })
// .Select(g => new
// {
// g.Key, Count = g.Count()
// })
//
// .Execute(logger);
//
// logger.LogDebug("Aggregated Data {Result}", mongoQuery.ToJsonString());
//
// // Works
// var summary = new SingleSeriesDataset() {
// Values = mongoQuery
// .GroupBy(q => q.Key.Classification)
// .ToDictionary(
// g => g.Key,
// g => g.Sum(k => k.Count)
// )
// };
//
// var r = new SummarisedDataset<SingleSeriesDataset, StringBucketDimensionResult>()
// {
// Summary = summary,
// Result = mongoQuery.Select(a => new StringBucketDimensionResult()
// {
// Fields = new Dictionary<string, string>()
// {
// { "Classification", a.Key.Classification },
// { "CheckCode", a.Key.CheckCode! },
// { "AlvsDecisionCode", a.Key.AlvsDecisionCode! },
// { "BtmsDecisionCode", a.Key.BtmsDecisionCode! }
// },
// Value = a.Count
// })
// .OrderBy(r => r.Value)
// .Reverse()
// .ToList()
// };
//
// return Task.FromResult(r);
// }

private static Task<TabularDataset<ByNameDimensionResult>> DefaultTabularDatasetByNameDimensionResult()
{
Expand Down
3 changes: 2 additions & 1 deletion Btms.Backend/appsettings.Development.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,8 @@
"Credentials": {
"PhaService": "eFrSxkkTurBU",
"BtmsAdmin": "ebuzKbuuvdcm",
"DmpDeveloper": "AzELT2hz7sS5"
"DmpDeveloper": "AzELT2hz7sS5",
"BtmsFrontend": "a498HnJ79A5G"
}
}
}
7 changes: 4 additions & 3 deletions Btms.Backend/appsettings.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,9 +31,10 @@
},
"AuthKeyStore": {
"Credentials": {
"PhaService": "",
"BtmsAdmin": "",
"DmpDeveloper": ""
"PhaService": "",
"BtmsAdmin": "",
"DmpDeveloper": "",
"BtmsFrontend": ""
}
}
}
Loading