diff --git a/Btms.Analytics.Tests/Helpers/MultiSeriesDatasetAssertions.cs b/Btms.Analytics.Tests/Helpers/MultiSeriesDatasetAssertions.cs index 5c1df55..4f4ce7e 100644 --- a/Btms.Analytics.Tests/Helpers/MultiSeriesDatasetAssertions.cs +++ b/Btms.Analytics.Tests/Helpers/MultiSeriesDatasetAssertions.cs @@ -19,7 +19,7 @@ public void BeSameLength(string because = "", params object[] becauseArgs) [CustomAssertion] public void HaveResults(string because = "", params object[] becauseArgs) { - test!.Sum(d => d.Results.Sum(r => r.Value)) + test!.Sum(d => d.Results.Sum(r => ((ByNumericDimensionResult)r).Value)) .Should().BeGreaterThan(0); } } \ No newline at end of file diff --git a/Btms.Analytics.Tests/ImportNotificationsByVersionTests.cs b/Btms.Analytics.Tests/ImportNotificationsByVersionTests.cs new file mode 100644 index 0000000..936990b --- /dev/null +++ b/Btms.Analytics.Tests/ImportNotificationsByVersionTests.cs @@ -0,0 +1,54 @@ +using Btms.Common.Extensions; +using FluentAssertions; +using Xunit; +using Xunit.Abstractions; + +using Btms.Analytics.Tests.Fixtures; + +namespace Btms.Analytics.Tests; + +[Collection(nameof(BasicSampleDataTestCollection))] +public class ImportNotificationsByVersionTests( + BasicSampleDataTestFixture basicSampleDataTestFixture, + ITestOutputHelper testOutputHelper) +{ + + [Fact] + public async Task WhenCalledLastMonth_ReturnExpectedAggregation() + { + testOutputHelper.WriteLine("Querying for aggregated data"); + var result = (await basicSampleDataTestFixture.GetImportNotificationsAggregationService(testOutputHelper) + .ByMaxVersion(DateTime.Today.MonthAgo(), DateTime.Today.Tomorrow())); + + testOutputHelper.WriteLine("{0} aggregated items found", result.Values.Count); + + result.Values.Count.Should().Be(2); + } + + [Fact] + public async Task WhenCalledLast48Hours_ReturnExpectedAggregation() + { + testOutputHelper.WriteLine("Querying for aggregated data"); + var result = (await basicSampleDataTestFixture.GetImportNotificationsAggregationService(testOutputHelper) + .ByStatus(DateTime.Now.NextHour().AddDays(-2), DateTime.Now.NextHour())); + + testOutputHelper.WriteLine($"{result.Values.Count} aggregated items found"); + + result.Values.Count.Should().Be(8); + result.Values.Keys.Order().Should().Equal( + "CHEDA Linked", "CHEDA Not Linked", "CHEDD Linked", "CHEDD Not Linked", "CHEDP Linked", "CHEDP Not Linked", "CHEDPP Linked", "CHEDPP Not Linked"); + + } + + [Fact] + public async Task WhenCalledWithTimePeriodYieldingNoResults_ReturnEmptyAggregation() + { + testOutputHelper.WriteLine("Querying for aggregated data"); + var result = (await basicSampleDataTestFixture.GetImportNotificationsAggregationService(testOutputHelper) + .ByStatus(DateTime.MaxValue.AddDays(-1), DateTime.MaxValue)); + + testOutputHelper.WriteLine($"{result.Values.Count} aggregated items found"); + + result.Values.Count.Should().Be(8); + } +} \ No newline at end of file diff --git a/Btms.Analytics.Tests/MovementsByDecisionsTests.cs b/Btms.Analytics.Tests/MovementsByDecisionsTests.cs index 5e0e312..72a797e 100644 --- a/Btms.Analytics.Tests/MovementsByDecisionsTests.cs +++ b/Btms.Analytics.Tests/MovementsByDecisionsTests.cs @@ -20,13 +20,12 @@ public async Task WhenCalled_ReturnExpectedAggregation() testOutputHelper.WriteLine("Querying for aggregated data"); var result = (await multiItemDataTestFixture.GetMovementsAggregationService(testOutputHelper) .ByDecision(DateTime.Today.MonthAgo(), DateTime.Today.Tomorrow())) - .Values - .ToList(); + .Result; testOutputHelper.WriteLine("{0} aggregated items found", result.Count); - result.Count.Should().Be(4); - result.Select(r => r.Key).Order().Should() - .Equal("ALVS Linked : H01", "BTMS Linked : C03", "BTMS Linked : X00", "BTMS Not Linked : X00"); + result.Count.Should().BeGreaterThan(1); + // result.Select(r => r.Key).Order().Should() + // .Equal("ALVS Linked : H01", "BTMS Linked : C03", "BTMS Linked : X00", "BTMS Not Linked : X00"); } } \ No newline at end of file diff --git a/Btms.Analytics.Tests/MovementsByUniqueDocumentReferenceTests.cs b/Btms.Analytics.Tests/MovementsByUniqueDocumentReferenceTests.cs index f7a17e2..507aabd 100644 --- a/Btms.Analytics.Tests/MovementsByUniqueDocumentReferenceTests.cs +++ b/Btms.Analytics.Tests/MovementsByUniqueDocumentReferenceTests.cs @@ -32,7 +32,7 @@ public async Task WhenCalledLastWeek_ReturnExpectedAggregation() { d.Dimension.Should().Be("Document Reference Count"); d.Results.Count().Should().NotBe(0); - d.Results.Sum(r => r.Value).Should().BeGreaterThan(0); + d.Results.Sum(r => ((ByNumericDimensionResult)r).Value).Should().BeGreaterThan(0); }); result.Should().HaveResults(); diff --git a/Btms.Analytics.Tests/MovementsExceptionsTests.cs b/Btms.Analytics.Tests/MovementsExceptionsTests.cs new file mode 100644 index 0000000..2d574f9 --- /dev/null +++ b/Btms.Analytics.Tests/MovementsExceptionsTests.cs @@ -0,0 +1,31 @@ +using Btms.Common.Extensions; +using FluentAssertions; +using Xunit; +using Xunit.Abstractions; + +using Btms.Analytics.Tests.Fixtures; +using Btms.Analytics.Tests.Helpers; + +namespace Btms.Analytics.Tests; + +[Collection(nameof(MultiItemDataTestCollection))] +public class MovementsExceptionsTests( + MultiItemDataTestFixture multiItemDataTestFixture, + ITestOutputHelper testOutputHelper) +{ + + [Fact] + public async Task WhenCalled_ReturnExpectedAggregation() + { + testOutputHelper.WriteLine("Querying for aggregated data"); + var result = (await multiItemDataTestFixture.GetMovementsAggregationService(testOutputHelper) + .ByDecision(DateTime.Today.MonthAgo(), DateTime.Today.Tomorrow())) + .Result; + + testOutputHelper.WriteLine("{0} aggregated items found", result.Count); + + result.Count.Should().BeGreaterThan(1); + // result.Select(r => r.Key).Order().Should() + // .Equal("ALVS Linked : H01", "BTMS Linked : C03", "BTMS Linked : X00", "BTMS Not Linked : X00"); + } +} \ No newline at end of file diff --git a/Btms.Analytics/Results.cs b/Btms.Analytics/Dataset.cs similarity index 61% rename from Btms.Analytics/Results.cs rename to Btms.Analytics/Dataset.cs index dc228a2..65444fd 100644 --- a/Btms.Analytics/Results.cs +++ b/Btms.Analytics/Dataset.cs @@ -1,27 +1,51 @@ using System.Diagnostics.CodeAnalysis; using System.Text.Json; using System.Text.Json.Serialization; -using Btms.Model.Auditing; namespace Btms.Analytics; -public class ByDateTimeResult +// A marker interface to identify things we want to be able to return from the analytics API +public interface IDataset; + +public class SummarisedDataset : IDataset + where TResult : IDimensionResult + where TSummary : IDimensionResult { - public DateTime Period { get; set; } - public int Value { get; set; } + public required TSummary Summary { get; set; } + public required List Result { get; set; } + } -public class ByNumericDimensionResult +public class MultiSeriesDataset : IDataset { - public int Dimension { get; set; } - public int Value { get; set; } + public List Series { get; set; } = []; +} + +public class SingleSeriesDataset : IDataset, IDimensionResult +{ + public IDictionary Values { get; set; } = new Dictionary(); +} + +public class TabularDataset : IDataset where TColumn : IDimensionResult +{ + public required List> Rows { get; set; } +} + +public class EntityDataset(IEnumerable items) : IDataset +{ + public IEnumerable Items { get; set; } = items; +} + +public class MultiSeriesDatetimeDataset : IDataset +{ + public List Series { get; set; } = []; } /// /// Serialise the derived types of IDataset /// /// -public class ResultTypeMappingConverter : JsonConverter where TType : IDataset +public class DatasetResultTypeMappingConverter : JsonConverter where TType : IDataset { [return: MaybeNull] public override TType Read( @@ -42,52 +66,17 @@ public override void Write(Utf8JsonWriter writer, TType value, JsonSerializerOpt { JsonSerializer.Serialize(writer, value as SingleSeriesDataset, options); } + else if (value is TabularDataset) + { + JsonSerializer.Serialize(writer, value as TabularDataset, options); + } + else if (value is SummarisedDataset) + { + JsonSerializer.Serialize(writer, value as SummarisedDataset, options); + } else { throw new NotImplementedException(); } } -} - -// A marker interface to identify things we want to be able to return from the analytics API -public interface IDataset; - -public class SingleSeriesDataset : IDataset -{ - public IDictionary Values { get; set; } = new Dictionary(); -} - -public class AuditHistory(AuditEntry auditEntry, string resourceType, string resourceId) -{ - public AuditEntry AuditEntry { get; set; } = auditEntry; - public string ResourceType { get; set; } = resourceType; - public string ResourceId { get; set; } = resourceId; -} - -public class EntityDataset(IEnumerable items) : IDataset -{ - public IEnumerable Items { get; set; } = items; -} - -public class MultiSeriesDatetimeDataset : IDataset -{ - public List Series { get; set; } = []; -} - -public class DatetimeSeries(string name) -{ - public string Name { get; set; } = name; - public List Periods { get; set; } = []; -} - -public class MultiSeriesDataset : IDataset -{ - public List Series { get; set; } = []; -} - -public class Series(string name, string dimension) -{ - public string Name { get; set; } = name; - public string Dimension { get; set; } = dimension; - public List Results { get; set; } = []; } \ No newline at end of file diff --git a/Btms.Analytics/DatasetDimensions.cs b/Btms.Analytics/DatasetDimensions.cs new file mode 100644 index 0000000..6a0d4ca --- /dev/null +++ b/Btms.Analytics/DatasetDimensions.cs @@ -0,0 +1,104 @@ +using System.Diagnostics.CodeAnalysis; +using System.Text.Json; +using System.Text.Json.Serialization; +using Btms.Model.Auditing; + +namespace Btms.Analytics; + +public class ByDateTimeResult +{ + public DateTime Period { get; set; } + public int Value { get; set; } +} + +public class ExceptionResult : IDimensionResult +{ + public required string Id { get; set; } + public required string Resource { get; set; } + public required DateTime UpdatedSource { get; set; } + public required DateTime Updated { get; set; } + + public required int ItemCount { get; set; } + public required int MaxEntryVersion { get; set; } + public required int MaxDecisionNumber { get; set; } + public required int LinkedCheds { get; set; } + public required string Reason { get; set; } +} + +public interface IDimensionResult; + +public class TabularDimensionRow where TColumn : IDimensionResult +{ + public required string Key { get; set; } + public required List Columns { get; set; } +} + +public class ByNumericDimensionResult : IDimensionResult +{ + public int Dimension { get; set; } + public int Value { get; set; } +} + +public class ByNameDimensionResult : IDimensionResult +{ + public required string Name { get; set; } + public int Value { get; set; } +} + +public class StringBucketDimensionResult : IDimensionResult +{ + public required Dictionary Fields { get; set; } + public int Value { get; set; } +} + +public class AuditHistory(AuditEntry auditEntry, string resourceType, string resourceId) +{ + public AuditEntry AuditEntry { get; set; } = auditEntry; + public string ResourceType { get; set; } = resourceType; + public string ResourceId { get; set; } = resourceId; +} + +public class DatetimeSeries(string name) +{ + public string Name { get; set; } = name; + public List Periods { get; set; } = []; +} + +public class Series +{ + public required string Name { get; set; } + public required string Dimension { get; set; } + public required List Results { get; set; } +} + +/// +/// Serialise the derived types of IDimensionResult +/// +/// +public class DimensionResultTypeMappingConverter : JsonConverter where TType : IDimensionResult +{ + [return: MaybeNull] + public override TType Read( + ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) => + throw new NotImplementedException(); + + public override void Write(Utf8JsonWriter writer, TType value, JsonSerializerOptions options) + { + if (value is ByNumericDimensionResult) + { + JsonSerializer.Serialize(writer, value as ByNumericDimensionResult, options); + } + else if (value is ByNameDimensionResult) + { + JsonSerializer.Serialize(writer, value as ByNameDimensionResult, options); + } + else if (value is ByNameDimensionResult) + { + JsonSerializer.Serialize(writer, value as ByNameDimensionResult, options); + } + else + { + throw new NotImplementedException(); + } + } +} \ No newline at end of file diff --git a/Btms.Analytics/Extensions/AnalyticsExtensions.cs b/Btms.Analytics/Extensions/AnalyticsExtensions.cs index 76ed099..afacd5f 100644 --- a/Btms.Analytics/Extensions/AnalyticsExtensions.cs +++ b/Btms.Analytics/Extensions/AnalyticsExtensions.cs @@ -1,3 +1,4 @@ +using System.Collections.Immutable; using Btms.Backend.Data; using Btms.Model.Data; using Microsoft.Extensions.Configuration; @@ -85,6 +86,16 @@ public static T[] AsOrderedArray(this IEnumerable en, Func .ToArray(); } + /// + /// Gives us an opportunity to hook into the mongo execution and + /// grab the query as a string after it's been executed + /// + /// + /// + /// + /// + /// + /// internal static IEnumerable> Execute(this IQueryable> source, ILogger logger) { try @@ -103,6 +114,108 @@ internal static IEnumerable> Execute(thi } } + /// + /// Gives us an opportunity to hook into the mongo execution and + /// grab the query as a string after it's been executed + /// + /// + /// + /// + /// + /// + /// + internal static IEnumerable Execute(this IAggregateFluent source, ILogger logger) + { + try + { + var aggregatedData = source.ToList(); + return aggregatedData; + } + catch(Exception ex) + { + logger.LogError(ex, "Error querying Mongo : {Message}", ex.Message); + throw new AnalyticsException("Error querying Mongo", ex); + } + finally + { + logger.LogInformation("Query from IAggregateFluent"); + // logger.LogExecutedMongoString((IQueryable)source); + } + } + + /// + /// Gives us an opportunity to hook into the mongo execution and + /// grab the query as a string after it's been executed + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + public static Dictionary ExecuteAsDictionary( + this IEnumerable source, + ILogger logger, + Func keySelector, + Func elementSelector) + where TKey : notnull + { + try + { + var aggregatedData = source + .ToDictionary(keySelector, elementSelector); + return aggregatedData; + } + catch(Exception ex) + { + logger.LogError(ex, "Error querying Mongo : {Message}", ex.Message); + throw new AnalyticsException("Error querying Mongo", ex); + } + finally + { + logger.LogExecutedMongoString((IQueryable)source); + } + } + /// + /// Gives us an opportunity to hook into the mongo execution and + /// grab the query as a string after it's been executed + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + public static ImmutableSortedDictionary ExecuteAsSortedDictionary( + this IEnumerable source, + ILogger logger, + Func keySelector, + Func elementSelector) + where TKey : notnull + { + try + { + var aggregatedData = source + .ToImmutableSortedDictionary(keySelector, elementSelector); + return aggregatedData; + } + catch(Exception ex) + { + logger.LogError(ex, "Error querying Mongo : {Message}", ex.Message); + throw new AnalyticsException("Error querying Mongo", ex); + } + finally + { + logger.LogExecutedMongoString((IQueryable)source); + } + } + internal static IEnumerable Execute( this IQueryable source, ILogger logger) { @@ -132,10 +245,10 @@ internal static IEnumerable Execute( private static void LogExecutedMongoString(this ILogger logger, IQueryable source) { var stages = ((IMongoQueryProvider)source.Provider).LoggedStages; - - logger.LogInformation("[{Query}]", string.Join(",", stages.Select(s => s.ToString()).ToArray())); + var query = string.Join(",", stages.Select(s => s.ToString()).ToArray()); + logger.LogInformation("[{Query}]", query); } - + public static async Task AsIDataset(this Task ms) { await ms; @@ -147,10 +260,24 @@ public static async Task AsIDataset(this Task ms) await ms; return (IDataset)ms.Result; } + + public static async Task AsIDataset(this Task> ms) + { + await ms; + return (IDataset)ms.Result; + } public static async Task AsIDataset(this Task ms) { await ms; return (IDataset)ms.Result; } + + public static async Task AsIDataset(this Task> ms) + where TResult : IDimensionResult + where TSummary : IDimensionResult + { + await ms; + return (IDataset)ms.Result; + } } \ No newline at end of file diff --git a/Btms.Analytics/IImportNotificationsAggregationService.cs b/Btms.Analytics/IImportNotificationsAggregationService.cs index a6b4063..4ccdee3 100644 --- a/Btms.Analytics/IImportNotificationsAggregationService.cs +++ b/Btms.Analytics/IImportNotificationsAggregationService.cs @@ -6,4 +6,5 @@ public interface IImportNotificationsAggregationService public Task ByArrival(DateTime from, DateTime to, AggregationPeriod aggregateBy = AggregationPeriod.Day); public Task ByStatus(DateTime from, DateTime to); public Task ByCommodityCount(DateTime from, DateTime to); + public Task ByMaxVersion(DateTime from, DateTime to); } \ No newline at end of file diff --git a/Btms.Analytics/IMovementsAggregationService.cs b/Btms.Analytics/IMovementsAggregationService.cs index 948c8ec..a13deba 100644 --- a/Btms.Analytics/IMovementsAggregationService.cs +++ b/Btms.Analytics/IMovementsAggregationService.cs @@ -7,9 +7,14 @@ public interface IMovementsAggregationService public Task ByCreated(DateTime from, DateTime to, AggregationPeriod aggregateBy = AggregationPeriod.Day); public Task ByStatus(DateTime from, DateTime to); public Task ByItemCount(DateTime from, DateTime to); - public Task ByDecision(DateTime from, DateTime to); + public Task> ByDecision(DateTime from, DateTime to); + // public Task> ByDecisionAndLinkStatus(DateTime from, DateTime to); public Task ByUniqueDocumentReferenceCount(DateTime from, DateTime to); public Task UniqueDocumentReferenceByMovementCount(DateTime from, DateTime to); public Task ByCheck(DateTime from, DateTime to); public Task?> GetHistory(string movementId); + public Task ByMaxVersion(DateTime from, DateTime to); + public Task ByMaxDecisionNumber(DateTime from, DateTime to); + public Task> GetExceptions(DateTime from, DateTime to); + } \ No newline at end of file diff --git a/Btms.Analytics/ImportNotificationsAggregationService.cs b/Btms.Analytics/ImportNotificationsAggregationService.cs index 26ed8f3..f083694 100644 --- a/Btms.Analytics/ImportNotificationsAggregationService.cs +++ b/Btms.Analytics/ImportNotificationsAggregationService.cs @@ -7,6 +7,7 @@ using MongoDB.Driver; using Btms.Analytics.Extensions; +using MongoDB.Driver.Linq; namespace Btms.Analytics; @@ -22,7 +23,7 @@ public Task ByCreated(DateTime from, DateTime to, Ag string CreateDatasetName(BsonDocument b) => AnalyticsHelpers.GetLinkedName(b["_id"]["linked"].ToBoolean(), b["_id"]["importNotificationType"].ToString()!.FromImportNotificationTypeEnumString()); - return Aggregate(dateRange, CreateDatasetName, matchFilter, "$createdSource", aggregateBy); + return AggregateByLinkedAndNotificationType(dateRange, CreateDatasetName, matchFilter, "$createdSource", aggregateBy); } public Task ByArrival(DateTime from, DateTime to, AggregationPeriod aggregateBy = AggregationPeriod.Day) @@ -35,7 +36,7 @@ public Task ByArrival(DateTime from, DateTime to, Ag string CreateDatasetName(BsonDocument b) => AnalyticsHelpers.GetLinkedName(b["_id"]["linked"].ToBoolean(), b["_id"]["importNotificationType"].ToString()!.FromImportNotificationTypeEnumString()); - return Aggregate(dateRange, CreateDatasetName, matchFilter, "$partOne.arrivesAt", aggregateBy); + return AggregateByLinkedAndNotificationType(dateRange, CreateDatasetName, matchFilter, "$partOne.arrivesAt", aggregateBy); } public Task ByStatus(DateTime from, DateTime to) @@ -63,7 +64,11 @@ public Task ByCommodityCount(DateTime from, DateTime to) { ImportNotificationType = n.ImportNotificationType!.Value, Linked = n.Relationships.Movements.Data.Count > 0, - CommodityCount = n.Commodities.Count() + + // ReSharper disable once ConditionIsAlwaysTrueOrFalseAccordingToNullableAPIContract + // This is not nullable, but is being represented as null in mongo and so this linq + // query needs to consider the null + CommodityCount = n.Commodities == null ? 0 : n.Commodities.Count() }) .Select(g => new { g.Key, Count = g.Count() }); @@ -95,20 +100,39 @@ public Task ByCommodityCount(DateTime from, DateTime to) return Task.FromResult(new MultiSeriesDataset() { Series = AnalyticsHelpers.GetImportNotificationSegments() - .Select(title => new Series(title, "ItemCount") + .Select(title => new Series() { + Name = title, + Dimension = "ItemCount", Results = Enumerable.Range(0, maxCommodities) .Select(i => new ByNumericDimensionResult { Dimension = i, Value = asDictionary.GetValueOrDefault(new { Title=title, CommodityCount = i }) - }).ToList() + }).ToList() }) .ToList() }); } - private Task Aggregate(DateTime[] dateRange, Func createDatasetName, Expression> filter, string dateField, AggregationPeriod aggregateBy) + public Task ByMaxVersion(DateTime from, DateTime to) + { + var data = context + .Notifications + .Where(n => n.CreatedSource >= from && n.CreatedSource < to) + .GroupBy(n => new { MaxVersion = + n.AuditEntries.Where(a => a.CreatedBy == "Ipaffs").Max(a => a.Version ) + }) + .Select(g => new { MaxVersion = g.Key.MaxVersion ?? 0, Count = g.Count() }) + .ExecuteAsSortedDictionary(logger, g => g.MaxVersion, g => g.Count); + + return Task.FromResult(new SingleSeriesDataset + { + Values = data.ToDictionary(d => d.Key.ToString(), d => d.Value) + }); + } + + private Task AggregateByLinkedAndNotificationType(DateTime[] dateRange, Func createDatasetName, Expression> filter, string dateField, AggregationPeriod aggregateBy) { var truncateBy = aggregateBy == AggregationPeriod.Hour ? "hour" : "day"; diff --git a/Btms.Analytics/MovementsAggregationService.cs b/Btms.Analytics/MovementsAggregationService.cs index 9b14f6a..d7d856a 100644 --- a/Btms.Analytics/MovementsAggregationService.cs +++ b/Btms.Analytics/MovementsAggregationService.cs @@ -5,9 +5,12 @@ using Btms.Common.Extensions; using Btms.Model.Extensions; using Btms.Model; +using Btms.Model.Alvs; using Btms.Model.Auditing; +using Btms.Model.Ipaffs; using MongoDB.Bson; using MongoDB.Driver; +using MongoDB.Driver.Linq; namespace Btms.Analytics; @@ -68,14 +71,16 @@ public Task ByItemCount(DateTime from, DateTime to) return Task.FromResult(new MultiSeriesDataset() { Series = AnalyticsHelpers.GetMovementSegments() - .Select(title => new Series(title, "Item Count") + .Select(title => new Series() { + Name = title, + Dimension = "Item Count", Results = Enumerable.Range(0, maxCount + 1) .Select(i => new ByNumericDimensionResult { Dimension = i, Value = dictionary.GetValueOrDefault(new { Title=title, ItemCount = i }, 0) - }).ToList() + }).ToList() } ) .ToList() @@ -110,15 +115,17 @@ public Task ByUniqueDocumentReferenceCount(DateTime from, Da return Task.FromResult(new MultiSeriesDataset() { Series = AnalyticsHelpers.GetMovementSegments() - .Select(title => new Series(title, "Document Reference Count") + .Select(title => new Series() { + Name = title, + Dimension = "Document Reference Count", Results = Enumerable.Range(0, maxReferences + 1) .Select(i => new ByNumericDimensionResult { Dimension = i, Value = dictionary.GetValueOrDefault(new { Title = title, DocumentReferenceCount = i }, 0) - }).ToList() + }).ToList() }) .ToList() }); @@ -179,6 +186,102 @@ public Task UniqueDocumentReferenceByMovementCount(DateTime return new EntityDataset(entries); } + public Task ByMaxVersion(DateTime from, DateTime to) + { + var data = context + .Movements + .Where(n => n.CreatedSource >= from && n.CreatedSource < to) + .GroupBy(n => new { MaxVersion = + n.ClearanceRequests.Max(a => a.Header!.EntryVersionNumber ) + }) + .Select(g => new { MaxVersion = g.Key.MaxVersion ?? 0, Count = g.Count() }) + .ExecuteAsSortedDictionary(logger, g => g.MaxVersion, g => g.Count); + + return Task.FromResult(new SingleSeriesDataset + { + Values = data.ToDictionary(d => d.Key.ToString(), d => d.Value) + }); + } + + public Task ByMaxDecisionNumber(DateTime from, DateTime to) + { + var data = context + .Movements + .Where(n => n.CreatedSource >= from && n.CreatedSource < to) + .GroupBy(n => new { MaxVersion = + n.Decisions.Max(a => a.Header!.DecisionNumber ) + }) + .Select(g => new { MaxVersion = g.Key.MaxVersion ?? 0, Count = g.Count() }) + .ExecuteAsSortedDictionary(logger, g => g.MaxVersion, g => g.Count); + + return Task.FromResult(new SingleSeriesDataset + { + Values = data.ToDictionary(d => d.Key.ToString(), d => d.Value) + }); + } + + public Task> GetExceptions(DateTime from, DateTime to) + { + var mongoQuery = context + .Movements + .Select(m => new + { + Id = m.Id, + UpdatedSource = m.UpdatedSource, + Updated = m.Updated, + MaxDecisionNumber = m.Decisions.Max(d => d.Header!.DecisionNumber) ?? 0, + MaxEntryVersion = m.ClearanceRequests.Max(c => c.Header!.EntryVersionNumber) ?? 0, + LinkedCheds = m.Relationships.Notifications.Data.Count, + ItemCount = m.Items.Count + + }) + .Select(m => new + { + Id = m.Id, + UpdatedSource = m.UpdatedSource, + Updated = m.Updated, + MaxDecisionNumber = m.MaxDecisionNumber, + MaxEntryVersion = m.MaxEntryVersion, + LinkedCheds = m.LinkedCheds, + ItemCount = m.ItemCount, + Total = m.MaxDecisionNumber + m.MaxEntryVersion + m.LinkedCheds + m.ItemCount + }) + .OrderBy(a => -a.Total) + .Take(10) + .Execute(logger); + + var result = mongoQuery.Select(r => + new ExceptionResult() + { + Resource = "Movement", + Id = r.Id!, + UpdatedSource = r.UpdatedSource!.Value, + Updated = r.Updated, + ItemCount = r.ItemCount, + MaxEntryVersion = r.MaxEntryVersion, + MaxDecisionNumber = r.MaxDecisionNumber, + LinkedCheds = r.LinkedCheds, + Reason = "High Number Of Messages" + }).ToList(); + // var result = new TabularDataset() + // { + // Rows = mongoQuery.Select(r => + // new TabularDimensionRow() + // { + // Key = r.Id!, Columns = + // [ + // new ByNameDimensionResult() { Name = "ItemCount", Value = r.ItemCount }, + // new ByNameDimensionResult() { Name = "MaxEntryVersion", Value = r.MaxEntryVersion }, + // new ByNameDimensionResult() { Name = "MaxDecisionNumber", Value = r.MaxDecisionNumber }, + // new ByNameDimensionResult() { Name = "LinkedCheds", Value = r.LinkedCheds } + // ] + // } + // ).ToList() + // }; + + return Task.FromResult(result); + } + public Task ByCheck(DateTime from, DateTime to) { return Task.FromResult(new MultiSeriesDataset() ); @@ -209,40 +312,176 @@ private Task Aggregate(DateTime[] dateRange, Func ByDecision(DateTime from, DateTime to) + /// + /// Finds the most recent decision from Alvs and BTMS + /// + /// + /// + /// + public Task> ByDecision(DateTime from, DateTime to) { var mongoQuery = context .Movements + // .Aggregate() .Where(m => m.CreatedSource >= from && m.CreatedSource < to) - .SelectMany(m => m.Decisions.Select(d => new { Decision = d, Movement = m })) + .Select(m => new + { + MovementInfo = new + { + Id = m.Id, + UpdatedSource = m.UpdatedSource, + Updated = m.Updated, + Movement = m + }, + // ReSharper disable once ConditionIsAlwaysTrueOrFalseAccordingToNullableAPIContract + // Get the most recent decision record from both systems + AlvsDecision = (m.Decisions == null + ? null + : m.Decisions + .Where(d => d.ServiceHeader!.SourceSystem == "ALVS") + .OrderBy(d => d.ServiceHeader!.ServiceCalled) + .Reverse() + .FirstOrDefault()) + // Creates a default item & check so we don't lose + // it in the selectmany below + ?? new AlvsClearanceRequest() + { + Items = new [] + { + new Items() + { + Checks = new [] + { + new Check() + { + CheckCode = "XXX", + DecisionCode = "XXX" + } + } + } + } + }, + BtmsDecision = m.Decisions == null + ? null + : m.Decisions + .Where(d => d.ServiceHeader!.SourceSystem == "BTMS") + .OrderBy(d => d.ServiceHeader!.ServiceCalled) + .Reverse() + .FirstOrDefault() + }) .SelectMany(m => - m.Decision.Items!.Select(i => new { Decision = m.Decision, Movement = m.Movement, Item = i })) - .SelectMany(m => m.Item.Checks!.Select(c => new + m.AlvsDecision!.Items! + .SelectMany(i => + (i.Checks + ?? new[] { new Check() { CheckCode = "XXX", DecisionCode = "XXX" } }) + .Select(c => + new + { + m.MovementInfo, + AlvsDecisionInfo = c.CheckCode == "XXX" ? null : new + { + Decision = m.AlvsDecision, + DecisionNumber = m.AlvsDecision!.Header!.DecisionNumber, + EntryVersion = m.AlvsDecision!.Header!.EntryVersionNumber, + ItemNumber = i.ItemNumber, + CheckCode = c.CheckCode, + DecisionCode = c.DecisionCode, + }, + BtmsDecisionInfo = new + { + Decision = m.BtmsDecision, + DecisionCode = m.BtmsDecision == null || m.BtmsDecision.Items == null + ? null + : m.BtmsDecision.Items! + .First(bi => bi.ItemNumber == i.ItemNumber) + .Checks! + .First(ch => ch.CheckCode == c.CheckCode) + .DecisionCode + } + } + ) + ) + .Select(a => new + { + a.MovementInfo, + a.AlvsDecisionInfo, + a.BtmsDecisionInfo, + Classification = + a.BtmsDecisionInfo == null ? "Btms Decision Not Present" : + a.AlvsDecisionInfo == null ? "Alvs Decision Not Present" : + + // TODO : we may want to try to consider clearance request version as well as the decision code + a.BtmsDecisionInfo.DecisionCode == a.AlvsDecisionInfo.DecisionCode ? "Btms Made Same Decision As Alvs" : + a.MovementInfo.Movement.Decisions + .Any(d => d.Header!.DecisionNumber == 1) ? "Alvs Decision Version 1 Not Present" : + a.MovementInfo.Movement.ClearanceRequests + .Any(d => d.Header!.EntryVersionNumber == 1) ? "Alvs Clearance Request Version 1 Not Present" : + a.AlvsDecisionInfo.DecisionNumber == 1 && a.AlvsDecisionInfo.EntryVersion == 1 ? "Single Entry And Decision Version" : + a.BtmsDecisionInfo.DecisionCode != a.AlvsDecisionInfo.DecisionCode ? "Btms Made Different Decision To Alvs" : + "Further Classification Needed" + // "FurtherClassificationNeeded Check Code Is " + a.AlvsDecisionInfo.CheckCode + }) + ) + + // .Where(m => m.AlvsDecisionInfo == null) + // .Where(m => m.AlvsDecision!.Items!.Any(i => i.Checks!.Any(c => c.CheckCode == "XXX"))) + .GroupBy(check => new { - CheckCode = c.CheckCode, - DecisionCode = c.DecisionCode, - DecisionSourceSystem = m.Decision.ServiceHeader!.SourceSystem, - DecisionEntryReference = m.Decision.Header!.EntryReference, - DecisionEntryVersionNumber = m.Decision.Header!.EntryVersionNumber, - Movement = m.Movement.EntryReference, - MovementVersion = m.Movement.EntryVersionNumber, - HasLinks = m.Movement.Relationships.Notifications.Data.Count > 0, - ItemNumber = m.Item.ItemNumber - })) - .GroupBy(m => new { m.HasLinks, m.DecisionSourceSystem, m.DecisionCode }) - .Select(m => new { m.Key.HasLinks, m.Key.DecisionSourceSystem, m.Key.DecisionCode, Count = m.Count() }) - .ToList(); - - logger.LogInformation("Found {0} items", mongoQuery.Count); - logger.LogInformation(mongoQuery.ToJsonString()); + check.Classification, + check.AlvsDecisionInfo!.CheckCode, + AlvsDecisionCode = check.AlvsDecisionInfo!.DecisionCode, + BtmsDecisionCode=check.BtmsDecisionInfo!.DecisionCode + }) + .Select(g => new + { + g.Key, Count = g.Count() + }) + + .Execute(logger); - return Task.FromResult(new SingleSeriesDataset() - { + logger.LogDebug("Aggregated Data {Result}", mongoQuery.ToJsonString()); + + // Works + var summary = new SingleSeriesDataset() { Values = mongoQuery + .GroupBy(q => q.Key.Classification) .ToDictionary( - r => $"{ r.DecisionSourceSystem } { ( r.HasLinks ? "Linked" : "Not Linked" ) } : { r.DecisionCode }", - r => r.Count + g => g.Key, + g => g.Sum(k => k.Count) ) + }; + + var r = new SummarisedDataset() + { + Summary = summary, + Result = mongoQuery.Select(a => new StringBucketDimensionResult() + { + Fields = new Dictionary() + { + { "Classification", a.Key.Classification }, + { "CheckCode", a.Key.CheckCode! }, + { "AlvsDecisionCode", a.Key.AlvsDecisionCode! }, + { "BtmsDecisionCode", a.Key.BtmsDecisionCode! } + }, + Value = a.Count + }) + .OrderBy(r => r.Value) + .Reverse() + .ToList() + }; + + return Task.FromResult(r); + } + + private static Task> DefaultTabularDatasetByNameDimensionResult() + { + return Task.FromResult(new TabularDataset() + { + Rows = + [ + new TabularDimensionRow() { Key = "", Columns = [] } + ] }); } + } \ No newline at end of file diff --git a/Btms.Backend/Config/AnalyticsDashboards.cs b/Btms.Backend/Config/AnalyticsDashboards.cs index ef4c1c0..ac405a8 100644 --- a/Btms.Backend/Config/AnalyticsDashboards.cs +++ b/Btms.Backend/Config/AnalyticsDashboards.cs @@ -71,8 +71,20 @@ public static async Task> GetCharts( () => importService.ByCommodityCount(DateTime.Today.MonthAgo(), DateTime.Now).AsIDataset() }, { - "lastMonthDecisionsByStatus", + "lastMonthsDecisionsByDecisionCode", () => movementsService.ByDecision(DateTime.Today.MonthAgo(), DateTime.Now).AsIDataset() + }, + { + "allImportNotificationsByVersion", + () => importService.ByMaxVersion(DateTime.Today.AddMonths(-3), DateTime.Today).AsIDataset() + }, + { + "allMovementsByMaxEntryVersion", + () => movementsService.ByMaxVersion(DateTime.Today.AddMonths(-3), DateTime.Today).AsIDataset() + }, + { + "allMovementsByMaxDecisionNumber", + () => movementsService.ByMaxDecisionNumber(DateTime.Today.AddMonths(-3), DateTime.Today).AsIDataset() } }; diff --git a/Btms.Backend/Endpoints/AnalyticsEndpoints.cs b/Btms.Backend/Endpoints/AnalyticsEndpoints.cs index ac42f21..3a6c65a 100644 --- a/Btms.Backend/Endpoints/AnalyticsEndpoints.cs +++ b/Btms.Backend/Endpoints/AnalyticsEndpoints.cs @@ -17,6 +17,7 @@ public static void UseAnalyticsEndpoints(this IEndpointRouteBuilder app) app.MapGet(BaseRoute + "/dashboard", GetDashboard).AllowAnonymous(); app.MapGet(BaseRoute + "/record-current-state", RecordCurrentState).AllowAnonymous(); app.MapGet(BaseRoute + "/timeline", Timeline); + app.MapGet(BaseRoute + "/exceptions", Exceptions); } private static async Task Timeline( [FromServices] IImportNotificationsAggregationService importService, @@ -32,6 +33,19 @@ private static async Task Timeline( return Results.NotFound(); } + private static async Task Exceptions( + [FromServices] IMovementsAggregationService movementsService) + { + var result + = await movementsService.GetExceptions(DateTime.MinValue, DateTime.Today); + + if (result.HasValue()) + { + return TypedResults.Json(result); + } + + return Results.NotFound(); + } private static async Task RecordCurrentState( [FromServices] ImportNotificationMetrics importNotificationMetrics) @@ -54,7 +68,8 @@ private static async Task GetDashboard( PropertyNamingPolicy = JsonNamingPolicy.CamelCase, Converters = { - new ResultTypeMappingConverter() + new DatasetResultTypeMappingConverter(), + new DimensionResultTypeMappingConverter() } }; diff --git a/Btms.Model/Movement.cs b/Btms.Model/Movement.cs index 0b70088..80f555d 100644 --- a/Btms.Model/Movement.cs +++ b/Btms.Model/Movement.cs @@ -137,10 +137,14 @@ public bool MergeDecision(string path, AlvsClearanceRequest clearanceRequest) } var decisionAuditContext = new Dictionary>(); - decisionAuditContext.Add("movements", new Dictionary() + decisionAuditContext.Add("clearanceRequests", new Dictionary() { { clearanceRequest.Header!.EntryReference!, clearanceRequest.Header!.EntryVersionNumber!.ToString()! } }); + decisionAuditContext.Add("decisions", new Dictionary() + { + { clearanceRequest.Header!.EntryReference!, clearanceRequest.Header!.DecisionNumber!.ToString()! } + }); decisionAuditContext.Add("importNotifications", new Dictionary() { { "todo", "todo" }