Skip to content

Commit

Permalink
CDMS-200 includes dates & country filtering in movements analytics (#28)
Browse files Browse the repository at this point in the history
  • Loading branch information
craigedmunds authored Dec 23, 2024
1 parent a86a753 commit 272ebc9
Show file tree
Hide file tree
Showing 16 changed files with 392 additions and 258 deletions.
2 changes: 1 addition & 1 deletion Btms.Analytics/IImportNotificationsAggregationService.cs
Original file line number Diff line number Diff line change
Expand Up @@ -6,5 +6,5 @@ public interface IImportNotificationsAggregationService
public Task<MultiSeriesDatetimeDataset> ByArrival(DateTime from, DateTime to, AggregationPeriod aggregateBy = AggregationPeriod.Day);
public Task<SingleSeriesDataset> ByStatus(DateTime from, DateTime to);
public Task<MultiSeriesDataset> ByCommodityCount(DateTime from, DateTime to);
public Task<SingleSeriesDataset> ByMaxVersion(DateTime from, DateTime to);
public Task<SingleSeriesDataset> ByMaxVersion(DateTime from, DateTime to, string[]? chedTypes = null, string? country = null);
}
11 changes: 6 additions & 5 deletions Btms.Analytics/IMovementsAggregationService.cs
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,15 @@ public interface IMovementsAggregationService
public Task<MultiSeriesDatetimeDataset> ByCreated(DateTime from, DateTime to, AggregationPeriod aggregateBy = AggregationPeriod.Day);
public Task<SingleSeriesDataset> ByStatus(DateTime from, DateTime to);
public Task<MultiSeriesDataset> ByItemCount(DateTime from, DateTime to);
public Task<SummarisedDataset<SingleSeriesDataset, StringBucketDimensionResult>> ByDecision(DateTime from, DateTime to);
public Task<SummarisedDataset<SingleSeriesDataset, StringBucketDimensionResult>> ByDecision(DateTime from, DateTime to, string[]? chedTypes = null, string? country = null);
// public Task<TabularDataset<ByNameDimensionResult>> ByDecisionAndLinkStatus(DateTime from, DateTime to);
public Task<MultiSeriesDataset> ByUniqueDocumentReferenceCount(DateTime from, DateTime to);
public Task<SingleSeriesDataset> UniqueDocumentReferenceByMovementCount(DateTime from, DateTime to);
public Task<MultiSeriesDataset> ByCheck(DateTime from, DateTime to);
// public Task<MultiSeriesDataset> ByCheck(DateTime from, DateTime to, string[]? chedTypes = null, string? country = null);
public Task<EntityDataset<AuditHistory>?> GetHistory(string movementId);
public Task<SingleSeriesDataset> ByMaxVersion(DateTime from, DateTime to);
public Task<SingleSeriesDataset> ByMaxDecisionNumber(DateTime from, DateTime to);
public Task<List<ExceptionResult>> GetExceptions(DateTime from, DateTime to);
public Task<SingleSeriesDataset> ByMaxVersion(DateTime from, DateTime to, string[]? chedTypes = null, string? country = null);
public Task<SingleSeriesDataset> ByMaxDecisionNumber(DateTime from, DateTime to, string[]? chedTypes = null, string? country = null);
public Task<List<ExceptionResult>> GetExceptions(DateTime from, DateTime to, string[]? chedTypes = null, string? country = null);
public Task<SingleSeriesDataset> ExceptionSummary(DateTime from, DateTime to, string[]? chedTypes = null, string? country = null);

}
3 changes: 2 additions & 1 deletion Btms.Analytics/ImportNotificationsAggregationService.cs
Original file line number Diff line number Diff line change
Expand Up @@ -115,11 +115,12 @@ public Task<MultiSeriesDataset> ByCommodityCount(DateTime from, DateTime to)
});
}

public Task<SingleSeriesDataset> ByMaxVersion(DateTime from, DateTime to)
public Task<SingleSeriesDataset> ByMaxVersion(DateTime from, DateTime to, string[]? chedTypes = null, string? country = null)
{
var data = context
.Notifications
.Where(n => n.CreatedSource >= from && n.CreatedSource < to)
.Where(m => country == null || m.PartOne!.Route!.TransitingStates!.Contains(country))
.GroupBy(n => new { MaxVersion =
n.AuditEntries.Where(a => a.CreatedBy == "Ipaffs").Max(a => a.Version )
})
Expand Down
109 changes: 109 additions & 0 deletions Btms.Analytics/MovementExceptions.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,109 @@
using Btms.Analytics.Extensions;
using Btms.Backend.Data;
using Microsoft.Extensions.Logging;

namespace Btms.Analytics;

public class MovementExceptions(IMongoDbContext context, ILogger logger)
{
//Returns a summary of the exceptions or a list
// Means we can share the same anonymous / query code without needing to create loads
// of classes
public (SingleSeriesDataset summary, List<ExceptionResult>) GetAllExceptions(DateTime from, DateTime to, bool summary, string[]? chedTypes = null, string? country = null)
{
var exceptionsSummary = new SingleSeriesDataset();
var exceptionsResult = new List<ExceptionResult>();

var simplifiedMovementView = context
.Movements
.Where(n => n.CreatedSource >= from && n.CreatedSource < to)
.Where(m => country == null || m.DispatchCountryCode == country )
.Select(m => new
{
Id = m.Id,
UpdatedSource = m.UpdatedSource,
Updated = m.Updated,
MaxDecisionNumber = m.Decisions.Max(d => d.Header!.DecisionNumber) ?? 0,
MaxEntryVersion = m.ClearanceRequests.Max(c => c.Header!.EntryVersionNumber) ?? 0,
LinkedCheds = m.Relationships.Notifications.Data.Count,
ItemCount = m.Items.Count,
HasMatchDecisions = m.AlvsDecisions.Any(d => d.Context.AlvsAnyMatch),
HasNotificationRelationships = m.Relationships.Notifications.Data.Count > 0
})
.Select(m => new
{
Id = m.Id,
UpdatedSource = m.UpdatedSource,
Updated = m.Updated,
MaxDecisionNumber = m.MaxDecisionNumber,
MaxEntryVersion = m.MaxEntryVersion,
LinkedCheds = m.LinkedCheds,
ItemCount = m.ItemCount,
HasMatchDecisions = m.HasMatchDecisions,
HasNotificationRelationships = m.HasNotificationRelationships,
Total = m.MaxDecisionNumber + m.MaxEntryVersion + m.LinkedCheds + m.ItemCount,
// TODO - can we include CHED versions here too?
TotalDocumentVersions = m.MaxDecisionNumber + m.MaxEntryVersion + m.LinkedCheds
});

var moreComplexMovementsQuery = simplifiedMovementView
.Where(r => r.TotalDocumentVersions > 5);

if (summary)
{
exceptionsSummary.Values.Add("complexMovement", moreComplexMovementsQuery.Count());
}
else
{
exceptionsResult.AddRange(moreComplexMovementsQuery
.OrderBy(a => -a.Total)
.Take(10)
.Execute(logger)
.Select(r =>
new ExceptionResult()
{
Resource = "Movement",
Id = r.Id!,
UpdatedSource = r.UpdatedSource!.Value,
Updated = r.Updated,
ItemCount = r.ItemCount,
MaxEntryVersion = r.MaxEntryVersion,
MaxDecisionNumber = r.MaxDecisionNumber,
LinkedCheds = r.LinkedCheds,
Reason = "High Number Of Messages"
})
);
}

var movementsWhereAlvsLinksButNotBtmsQuery = simplifiedMovementView
.Where(r => r.HasMatchDecisions && !r.HasNotificationRelationships);

if (summary)
{
exceptionsSummary.Values.Add("alvsLinksButNotBtms", movementsWhereAlvsLinksButNotBtmsQuery.Count());
}
else
{
exceptionsResult.AddRange(movementsWhereAlvsLinksButNotBtmsQuery
.OrderBy(a => -a.Total)
.Take(10)
.Execute(logger)
.Select(r =>
new ExceptionResult()
{
Resource = "Movement",
Id = r.Id!,
UpdatedSource = r.UpdatedSource!.Value,
Updated = r.Updated,
ItemCount = r.ItemCount,
MaxEntryVersion = r.MaxEntryVersion,
MaxDecisionNumber = r.MaxDecisionNumber,
LinkedCheds = r.LinkedCheds,
Reason = "Alvs has match decisions but we don't have links"
})
);
}

return (exceptionsSummary, exceptionsResult);
}
}
Loading

0 comments on commit 272ebc9

Please sign in to comment.