diff --git a/Btms.Analytics.Tests/Helpers/MultiSeriesDatasetAssertions.cs b/Btms.Analytics.Tests/Helpers/MultiSeriesDatasetAssertions.cs index 85350c7..5c1df55 100644 --- a/Btms.Analytics.Tests/Helpers/MultiSeriesDatasetAssertions.cs +++ b/Btms.Analytics.Tests/Helpers/MultiSeriesDatasetAssertions.cs @@ -3,8 +3,8 @@ namespace Btms.Analytics.Tests.Helpers; -public class MultiSeriesDatasetAssertions(List? test) - : GenericCollectionAssertions(test) +public class MultiSeriesDatasetAssertions(List? test) + : GenericCollectionAssertions(test) { [CustomAssertion] public void BeSameLength(string because = "", params object[] becauseArgs) diff --git a/Btms.Analytics.Tests/Helpers/SingleSeriesDatasetAssertions.cs b/Btms.Analytics.Tests/Helpers/SingleSeriesDatasetAssertions.cs index e8ff273..dbc212b 100644 --- a/Btms.Analytics.Tests/Helpers/SingleSeriesDatasetAssertions.cs +++ b/Btms.Analytics.Tests/Helpers/SingleSeriesDatasetAssertions.cs @@ -2,7 +2,7 @@ namespace Btms.Analytics.Tests.Helpers; -public class SingleSeriesDatasetAssertions(SingeSeriesDataset? test) +public class SingleSeriesDatasetAssertions(SingleSeriesDataset? test) { [CustomAssertion] public void HaveResults(string because = "", params object[] becauseArgs) diff --git a/Btms.Analytics.Tests/Helpers/TestAssertionExtensions.cs b/Btms.Analytics.Tests/Helpers/TestAssertionExtensions.cs index 2fbd224..3e21166 100644 --- a/Btms.Analytics.Tests/Helpers/TestAssertionExtensions.cs +++ b/Btms.Analytics.Tests/Helpers/TestAssertionExtensions.cs @@ -2,11 +2,11 @@ namespace Btms.Analytics.Tests.Helpers; public static class TestAssertionExtensions { - public static MultiSeriesDatasetAssertions Should(this List? instance) + public static MultiSeriesDatasetAssertions Should(this List? instance) { return new MultiSeriesDatasetAssertions(instance); } - public static SingleSeriesDatasetAssertions Should(this SingeSeriesDataset? instance) + public static SingleSeriesDatasetAssertions Should(this SingleSeriesDataset? instance) { return new SingleSeriesDatasetAssertions(instance); } diff --git a/Btms.Analytics.Tests/ImportNotificationsByArrivalDateTests.cs b/Btms.Analytics.Tests/ImportNotificationsByArrivalDateTests.cs index cfc4a53..61c9baf 100644 --- a/Btms.Analytics.Tests/ImportNotificationsByArrivalDateTests.cs +++ b/Btms.Analytics.Tests/ImportNotificationsByArrivalDateTests.cs @@ -20,6 +20,7 @@ public async Task WhenCalledNextMonth_ReturnExpectedAggregation() var result = (await basicSampleDataTestFixture.ImportNotificationsAggregationService .ByArrival(DateTime.Today, DateTime.Today.MonthLater())) + .Series .ToList(); testOutputHelper.WriteLine($"{result.Count} aggregated items found"); diff --git a/Btms.Analytics.Tests/ImportNotificationsByCommoditiesTests.cs b/Btms.Analytics.Tests/ImportNotificationsByCommoditiesTests.cs index c132682..10ce1dc 100644 --- a/Btms.Analytics.Tests/ImportNotificationsByCommoditiesTests.cs +++ b/Btms.Analytics.Tests/ImportNotificationsByCommoditiesTests.cs @@ -20,6 +20,7 @@ public async Task WhenCalledLastWeek_ReturnExpectedAggregation() testOutputHelper.WriteLine("Querying for aggregated data"); var result = (await multiItemDataTestFixture.ImportNotificationsAggregationService .ByCommodityCount(DateTime.Today.WeekAgo(), DateTime.Today.Tomorrow())) + .Series .ToList(); testOutputHelper.WriteLine("{0} aggregated items found", result.Count); diff --git a/Btms.Analytics.Tests/ImportNotificationsByCreatedDateTests.cs b/Btms.Analytics.Tests/ImportNotificationsByCreatedDateTests.cs index 8426b5f..de4e3f1 100644 --- a/Btms.Analytics.Tests/ImportNotificationsByCreatedDateTests.cs +++ b/Btms.Analytics.Tests/ImportNotificationsByCreatedDateTests.cs @@ -18,6 +18,7 @@ public async Task WhenCalledLast48Hours_ReturnExpectedAggregation() { var result = (await basicSampleDataTestFixture.ImportNotificationsAggregationService .ByCreated(DateTime.Now.NextHour().AddDays(-2), DateTime.Now.NextHour(),AggregationPeriod.Hour)) + .Series .ToList(); testOutputHelper.WriteLine(result.ToJsonString()); @@ -36,6 +37,7 @@ public async Task WhenCalledLastMonth_ReturnExpectedAggregation() { var result = (await basicSampleDataTestFixture.ImportNotificationsAggregationService .ByCreated(DateTime.Today.MonthAgo(), DateTime.Today.Tomorrow())) + .Series .ToList(); testOutputHelper.WriteLine(result.ToJsonString()); @@ -63,6 +65,7 @@ public async Task WhenCalledWithTimePeriodYieldingNoResults_ReturnEmptyAggregati var result = (await basicSampleDataTestFixture.ImportNotificationsAggregationService .ByCreated(from, to, AggregationPeriod.Hour)) + .Series .ToList(); testOutputHelper.WriteLine(result.ToJsonString()); diff --git a/Btms.Analytics.Tests/MovementHistoryTests.cs b/Btms.Analytics.Tests/MovementHistoryTests.cs new file mode 100644 index 0000000..8f356f7 --- /dev/null +++ b/Btms.Analytics.Tests/MovementHistoryTests.cs @@ -0,0 +1,28 @@ +using Btms.Common.Extensions; +using Xunit; +using Xunit.Abstractions; + +using Btms.Analytics.Tests.Fixtures; +using Btms.Analytics.Tests.Helpers; +using FluentAssertions; + +namespace Btms.Analytics.Tests; + +[Collection(nameof(MultiItemDataTestCollection))] +public class MovementHistoryTests( + MultiItemDataTestFixture multiItemDataTestFixture, + ITestOutputHelper testOutputHelper) +{ + [Fact] + public async Task WhenCalled_ReturnsHistory() + { + testOutputHelper.WriteLine("Querying for history"); + var result = await multiItemDataTestFixture.MovementsAggregationService + .GetHistory("23GB9999001215000001"); + + testOutputHelper.WriteLine("{0} history items found", result.Items.Count()); + + result.Items.Should().HasValue(); + result.Items.Select(a => a.AuditEntry.CreatedSource).Should().BeInAscendingOrder(); + } +} \ No newline at end of file diff --git a/Btms.Analytics.Tests/MovementsByCreatedDateTests.cs b/Btms.Analytics.Tests/MovementsByCreatedDateTests.cs index a490518..e0173b3 100644 --- a/Btms.Analytics.Tests/MovementsByCreatedDateTests.cs +++ b/Btms.Analytics.Tests/MovementsByCreatedDateTests.cs @@ -17,6 +17,7 @@ public async Task WhenCalledLast48Hours_ReturnExpectedAggregation() { var result = (await basicSampleDataTestFixture.MovementsAggregationService .ByCreated(DateTime.Now.NextHour().AddDays(-2), DateTime.Now.NextHour(), AggregationPeriod.Hour)) + .Series .ToList(); testOutputHelper.WriteLine(result.ToJsonString()); @@ -38,6 +39,7 @@ public async Task WhenCalledWithTimePeriodYieldingNoResults_ReturnEmptyAggregati var result = (await basicSampleDataTestFixture.MovementsAggregationService .ByCreated(from, to, AggregationPeriod.Hour)) + .Series .ToList(); testOutputHelper.WriteLine(result.ToJsonString()); @@ -62,6 +64,7 @@ public async Task WhenCalledLastMonth_ReturnExpectedAggregation() { var result = (await basicSampleDataTestFixture.MovementsAggregationService .ByCreated(DateTime.Today.MonthAgo(), DateTime.Today.Tomorrow())) + .Series .ToList(); testOutputHelper.WriteLine(result.ToJsonString()); diff --git a/Btms.Analytics.Tests/MovementsByItemsTests.cs b/Btms.Analytics.Tests/MovementsByItemsTests.cs index 1c2d9ce..477367a 100644 --- a/Btms.Analytics.Tests/MovementsByItemsTests.cs +++ b/Btms.Analytics.Tests/MovementsByItemsTests.cs @@ -20,6 +20,7 @@ public async Task WhenCalledLastWeek_ReturnExpectedAggregation() testOutputHelper.WriteLine("Querying for aggregated data"); var result = (await multiItemDataTestFixture.MovementsAggregationService .ByItemCount(DateTime.Today.WeekAgo(), DateTime.Today.Tomorrow())) + .Series .ToList(); testOutputHelper.WriteLine("{0} aggregated items found", result.Count); diff --git a/Btms.Analytics.Tests/MovementsByUniqueDocumentReferenceTests.cs b/Btms.Analytics.Tests/MovementsByUniqueDocumentReferenceTests.cs index b0b80be..52c61f3 100644 --- a/Btms.Analytics.Tests/MovementsByUniqueDocumentReferenceTests.cs +++ b/Btms.Analytics.Tests/MovementsByUniqueDocumentReferenceTests.cs @@ -20,6 +20,7 @@ public async Task WhenCalledLastWeek_ReturnExpectedAggregation() testOutputHelper.WriteLine("Querying for aggregated data"); var result = (await multiItemDataTestFixture.MovementsAggregationService .ByUniqueDocumentReferenceCount(DateTime.Today.WeekAgo(), DateTime.Today.Tomorrow())) + .Series .ToList(); testOutputHelper.WriteLine("{0} aggregated items found", result.Count); diff --git a/Btms.Analytics/Extensions/AnalyticsExtensions.cs b/Btms.Analytics/Extensions/AnalyticsExtensions.cs index de14f73..76ed099 100644 --- a/Btms.Analytics/Extensions/AnalyticsExtensions.cs +++ b/Btms.Analytics/Extensions/AnalyticsExtensions.cs @@ -28,7 +28,7 @@ public static IServiceCollection AddAnalyticsServices(this IServiceCollection se return services; } - public static string MetricsKey(this MultiSeriesDatetimeDataset ds) + public static string MetricsKey(this DatetimeSeries ds) { return ds.Name.Replace(" ", "-").ToLower(); } @@ -64,17 +64,15 @@ public static Dictionary GetNamedSetAsDict(this Dictionary records, DateTime[] dateRange, string title) + public static DatetimeSeries AsDataset(this Dictionary records, DateTime[] dateRange, + string title) { var dates = records.GetNamedSetAsDict(title); - return new MultiSeriesDatetimeDataset(title) + return new DatetimeSeries(title) { Periods = dateRange .Select(resultDate => - new ByDateTimeResult - { - Period = resultDate, Value = dates.GetValueOrDefault(resultDate, 0) - }) + new ByDateTimeResult { Period = resultDate, Value = dates.GetValueOrDefault(resultDate, 0) }) .Order(AnalyticsHelpers.ByDateTimeResultComparer) .ToList() }; @@ -89,7 +87,6 @@ public static T[] AsOrderedArray(this IEnumerable en, Func internal static IEnumerable> Execute(this IQueryable> source, ILogger logger) { - try { var aggregatedData = source.ToList(); @@ -138,4 +135,22 @@ private static void LogExecutedMongoString(this ILogger logger, IQueryable sourc logger.LogInformation("[{Query}]", string.Join(",", stages.Select(s => s.ToString()).ToArray())); } + + public static async Task AsIDataset(this Task ms) + { + await ms; + return (IDataset)ms.Result; + } + + public static async Task AsIDataset(this Task ms) + { + await ms; + return (IDataset)ms.Result; + } + + public static async Task AsIDataset(this Task ms) + { + await ms; + return (IDataset)ms.Result; + } } \ No newline at end of file diff --git a/Btms.Analytics/IImportNotificationsAggregationService.cs b/Btms.Analytics/IImportNotificationsAggregationService.cs index b768609..a6b4063 100644 --- a/Btms.Analytics/IImportNotificationsAggregationService.cs +++ b/Btms.Analytics/IImportNotificationsAggregationService.cs @@ -2,8 +2,8 @@ namespace Btms.Analytics; public interface IImportNotificationsAggregationService { - public Task ByCreated(DateTime from, DateTime to, AggregationPeriod aggregateBy = AggregationPeriod.Day); - public Task ByArrival(DateTime from, DateTime to, AggregationPeriod aggregateBy = AggregationPeriod.Day); - public Task ByStatus(DateTime from, DateTime to); - public Task ByCommodityCount(DateTime from, DateTime to); + public Task ByCreated(DateTime from, DateTime to, AggregationPeriod aggregateBy = AggregationPeriod.Day); + public Task ByArrival(DateTime from, DateTime to, AggregationPeriod aggregateBy = AggregationPeriod.Day); + public Task ByStatus(DateTime from, DateTime to); + public Task ByCommodityCount(DateTime from, DateTime to); } \ No newline at end of file diff --git a/Btms.Analytics/IMovementsAggregationService.cs b/Btms.Analytics/IMovementsAggregationService.cs index f9635ae..cd65eda 100644 --- a/Btms.Analytics/IMovementsAggregationService.cs +++ b/Btms.Analytics/IMovementsAggregationService.cs @@ -1,10 +1,14 @@ +using Btms.Model.Auditing; + namespace Btms.Analytics; public interface IMovementsAggregationService { - public Task ByCreated(DateTime from, DateTime to, AggregationPeriod aggregateBy = AggregationPeriod.Day); - public Task ByStatus(DateTime from, DateTime to); - public Task ByItemCount(DateTime from, DateTime to); - public Task ByUniqueDocumentReferenceCount(DateTime from, DateTime to); - public Task UniqueDocumentReferenceByMovementCount(DateTime from, DateTime to); + public Task ByCreated(DateTime from, DateTime to, AggregationPeriod aggregateBy = AggregationPeriod.Day); + public Task ByStatus(DateTime from, DateTime to); + public Task ByItemCount(DateTime from, DateTime to); + public Task ByUniqueDocumentReferenceCount(DateTime from, DateTime to); + public Task UniqueDocumentReferenceByMovementCount(DateTime from, DateTime to); + public Task ByCheck(DateTime from, DateTime to); + public Task> GetHistory(string movementId); } \ No newline at end of file diff --git a/Btms.Analytics/ImportNotificationMetrics.cs b/Btms.Analytics/ImportNotificationMetrics.cs index cf83903..d1cb2cb 100644 --- a/Btms.Analytics/ImportNotificationMetrics.cs +++ b/Btms.Analytics/ImportNotificationMetrics.cs @@ -39,14 +39,14 @@ public async Task RecordCurrentState() { var metrics = await _importService.ByCreated(DateTime.Today, DateTime.Now.NextHour()); - foreach (var dataset in metrics) + foreach (var series in metrics.Series) { - var key = $"{AnalyticsMetricNames.MetricPrefix}.import-notifications.{dataset.MetricsKey()}.count"; + var key = $"{AnalyticsMetricNames.MetricPrefix}.import-notifications.{series.MetricsKey()}.count"; if (_metrics.TryGetValue(key, out var instrument)) { if (instrument is Gauge g) { - g.Record(dataset.Periods[0].Value); + g.Record(series.Periods[0].Value); } else { diff --git a/Btms.Analytics/ImportNotificationsAggregationService.cs b/Btms.Analytics/ImportNotificationsAggregationService.cs index 48f622c..26ed8f3 100644 --- a/Btms.Analytics/ImportNotificationsAggregationService.cs +++ b/Btms.Analytics/ImportNotificationsAggregationService.cs @@ -12,7 +12,7 @@ namespace Btms.Analytics; public class ImportNotificationsAggregationService(IMongoDbContext context, ILogger logger) : IImportNotificationsAggregationService { - public Task ByCreated(DateTime from, DateTime to, AggregationPeriod aggregateBy = AggregationPeriod.Day) + public Task ByCreated(DateTime from, DateTime to, AggregationPeriod aggregateBy = AggregationPeriod.Day) { var dateRange = AnalyticsHelpers.CreateDateRange(from, to, aggregateBy); @@ -25,7 +25,7 @@ string CreateDatasetName(BsonDocument b) => return Aggregate(dateRange, CreateDatasetName, matchFilter, "$createdSource", aggregateBy); } - public Task ByArrival(DateTime from, DateTime to, AggregationPeriod aggregateBy = AggregationPeriod.Day) + public Task ByArrival(DateTime from, DateTime to, AggregationPeriod aggregateBy = AggregationPeriod.Day) { var dateRange = AnalyticsHelpers.CreateDateRange(from, to, aggregateBy); @@ -38,7 +38,7 @@ string CreateDatasetName(BsonDocument b) => return Aggregate(dateRange, CreateDatasetName, matchFilter, "$partOne.arrivesAt", aggregateBy); } - public Task ByStatus(DateTime from, DateTime to) + public Task ByStatus(DateTime from, DateTime to) { var data = context .Notifications @@ -48,13 +48,13 @@ public Task ByStatus(DateTime from, DateTime to) .ToDictionary(g => AnalyticsHelpers.GetLinkedName(g.Linked, g.ImportNotificationType.AsString()), g => g.Count); - return Task.FromResult(new SingeSeriesDataset + return Task.FromResult(new SingleSeriesDataset { Values = AnalyticsHelpers.GetImportNotificationSegments().ToDictionary(title => title, title => data.GetValueOrDefault(title, 0)) }); } - public Task ByCommodityCount(DateTime from, DateTime to) + public Task ByCommodityCount(DateTime from, DateTime to) { var query = context .Notifications @@ -72,8 +72,6 @@ public Task ByCommodityCount(DateTime from, DateTime to) .GroupBy(r => new { r.Key.ImportNotificationType, r.Key.Linked }) .ToList(); - // var maxCommodities = result.Max(r => r.Max(i => i.Key.CommodityCount)); - var maxCommodities = result.Count > 0 ? result.Max(r => r.Any() ? r.Max(i => i.Key.CommodityCount) : 0) : 0; @@ -94,24 +92,23 @@ public Task ByCommodityCount(DateTime from, DateTime to) g => g.NotificationCount); - return Task.FromResult( - AnalyticsHelpers.GetImportNotificationSegments() - .Select(title => new MultiSeriesDataset(title, "ItemCount") + return Task.FromResult(new MultiSeriesDataset() + { + Series = AnalyticsHelpers.GetImportNotificationSegments() + .Select(title => new Series(title, "ItemCount") { - // Results = asDictionary.AsResultList(title, maxCommodities) Results = Enumerable.Range(0, maxCommodities) .Select(i => new ByNumericDimensionResult { Dimension = i, Value = asDictionary.GetValueOrDefault(new { Title=title, CommodityCount = i }) }).ToList() - } - ) - .AsOrderedArray(d => d.Name) - ); + }) + .ToList() + }); } - private Task Aggregate(DateTime[] dateRange, Func createDatasetName, Expression> filter, string dateField, AggregationPeriod aggregateBy) + private Task Aggregate(DateTime[] dateRange, Func createDatasetName, Expression> filter, string dateField, AggregationPeriod aggregateBy) { var truncateBy = aggregateBy == AggregationPeriod.Hour ? "hour" : "day"; @@ -133,6 +130,6 @@ private Task Aggregate(DateTime[] dateRange, Func< logger.LogDebug("Aggregated Data {Result}", output.ToList().ToJsonString()); - return Task.FromResult(output); + return Task.FromResult(new MultiSeriesDatetimeDataset() { Series = output.ToList() }); } } \ No newline at end of file diff --git a/Btms.Analytics/MovementsAggregationService.cs b/Btms.Analytics/MovementsAggregationService.cs index 387da2a..7ce5169 100644 --- a/Btms.Analytics/MovementsAggregationService.cs +++ b/Btms.Analytics/MovementsAggregationService.cs @@ -4,6 +4,7 @@ using Btms.Backend.Data; using Btms.Model.Extensions; using Btms.Model; +using Btms.Model.Auditing; using MongoDB.Bson; using MongoDB.Driver; @@ -18,7 +19,7 @@ public class MovementsAggregationService(IMongoDbContext context, ILoggerTime period to search to (exclusive) /// Aggregate by day/hour /// - public Task ByCreated(DateTime from, DateTime to, AggregationPeriod aggregateBy = AggregationPeriod.Day) + public Task ByCreated(DateTime from, DateTime to, AggregationPeriod aggregateBy = AggregationPeriod.Day) { var dateRange = AnalyticsHelpers.CreateDateRange(from, to, aggregateBy); @@ -30,7 +31,7 @@ public Task ByCreated(DateTime from, DateTime to, return Aggregate(dateRange, CreateDatasetName, matchFilter, "$createdSource", aggregateBy); } - public Task ByStatus(DateTime from, DateTime to) + public Task ByStatus(DateTime from, DateTime to) { var data = context .Movements @@ -39,13 +40,13 @@ public Task ByStatus(DateTime from, DateTime to) .Select(g => new { g.Key, Count = g.Count() }) .ToDictionary(g => AnalyticsHelpers.GetLinkedName(g.Key), g => g.Count); - return Task.FromResult(new SingeSeriesDataset + return Task.FromResult(new SingleSeriesDataset { Values = AnalyticsHelpers.GetMovementSegments().ToDictionary(title => title, title => data.GetValueOrDefault(title, 0)) }); } - public Task ByItemCount(DateTime from, DateTime to) + public Task ByItemCount(DateTime from, DateTime to) { var mongoQuery = context .Movements @@ -63,20 +64,24 @@ public Task ByItemCount(DateTime from, DateTime to) var maxCount = mongoResult.Count > 0 ? mongoResult.Max(r => r.Count) : 0; - return Task.FromResult(AnalyticsHelpers.GetMovementSegments() - .Select(title => new MultiSeriesDataset(title, "Item Count") { - Results = Enumerable.Range(0, maxCount + 1) - .Select(i => new ByNumericDimensionResult + return Task.FromResult(new MultiSeriesDataset() + { + Series = AnalyticsHelpers.GetMovementSegments() + .Select(title => new Series(title, "Item Count") { - Dimension = i, - Value = dictionary.GetValueOrDefault(new { Title=title, ItemCount = i }, 0) - }).ToList() - }) - .ToArray() - ); + Results = Enumerable.Range(0, maxCount + 1) + .Select(i => new ByNumericDimensionResult + { + Dimension = i, + Value = dictionary.GetValueOrDefault(new { Title=title, ItemCount = i }, 0) + }).ToList() + } + ) + .ToList() + }); } - public Task ByUniqueDocumentReferenceCount(DateTime from, DateTime to) + public Task ByUniqueDocumentReferenceCount(DateTime from, DateTime to) { var mongoQuery = context .Movements @@ -100,21 +105,25 @@ public Task ByUniqueDocumentReferenceCount(DateTime from, var maxReferences = mongoResult.Count > 0 ? mongoResult.Max(r => r.DocumentReferenceCount) : 0; - - return Task.FromResult(AnalyticsHelpers.GetMovementSegments() - .Select(title => new MultiSeriesDataset(title, "Document Reference Count") { - Results = Enumerable.Range(0, maxReferences + 1) - .Select(i => new ByNumericDimensionResult - { - Dimension = i, - Value = dictionary.GetValueOrDefault(new { Title=title, DocumentReferenceCount = i }, 0) - }).ToList() - }) - .ToArray() - ); + + return Task.FromResult(new MultiSeriesDataset() + { + Series = AnalyticsHelpers.GetMovementSegments() + .Select(title => new Series(title, "Document Reference Count") + { + Results = Enumerable.Range(0, maxReferences + 1) + .Select(i => new ByNumericDimensionResult + { + Dimension = i, + Value = dictionary.GetValueOrDefault(new { Title = title, DocumentReferenceCount = i }, + 0) + }).ToList() + }) + .ToList() + }); } - public Task UniqueDocumentReferenceByMovementCount(DateTime from, DateTime to) + public Task UniqueDocumentReferenceByMovementCount(DateTime from, DateTime to) { var mongoQuery = context .Movements @@ -134,12 +143,42 @@ public Task UniqueDocumentReferenceByMovementCount(DateTime r =>r.MovementCount.ToString(), r=> r.DocumentReferenceCount); - var result = new SingeSeriesDataset { Values = mongoResult }; + var result = new SingleSeriesDataset { Values = mongoResult }; return Task.FromResult(result); } - private Task Aggregate(DateTime[] dateRange, Func createDatasetName, Expression> filter, string dateField, AggregationPeriod aggregateBy) + public async Task> GetHistory(string movementId) + { + var movement = await context + .Movements + .Find(movementId); + + var notificationIds = movement!.Relationships.Notifications.Data.Select(n => n.Id); + + var notificationEntries = context.Notifications + .Where(n => notificationIds.Contains(n.Id)) + .SelectMany(n => n.AuditEntries + .Select(a => + new AuditHistory(a, $"ImportNotification", n.Id!) + ) + ); + + var entries = movement!.AuditEntries + .Select(a => new AuditHistory(a, "Movement", movementId)) + .Concat(notificationEntries); + + entries = entries.OrderBy(a => a.AuditEntry.CreatedSource); + + return new EntityDataset(entries); + } + + public Task ByCheck(DateTime from, DateTime to) + { + return Task.FromResult(new MultiSeriesDataset() ); + } + + private Task Aggregate(DateTime[] dateRange, Func createDatasetName, Expression> filter, string dateField, AggregationPeriod aggregateBy) { var truncateBy = aggregateBy == AggregationPeriod.Hour ? "hour" : "day"; @@ -161,6 +200,6 @@ private Task Aggregate(DateTime[] dateRange, Func< logger.LogDebug("Aggregated Data {Result}", output.ToList().ToJsonString()); - return Task.FromResult(output); + return Task.FromResult(new MultiSeriesDatetimeDataset() { Series = output.ToList() }); } } \ No newline at end of file diff --git a/Btms.Analytics/Results.cs b/Btms.Analytics/Results.cs index 1411594..dc228a2 100644 --- a/Btms.Analytics/Results.cs +++ b/Btms.Analytics/Results.cs @@ -1,3 +1,8 @@ +using System.Diagnostics.CodeAnalysis; +using System.Text.Json; +using System.Text.Json.Serialization; +using Btms.Model.Auditing; + namespace Btms.Analytics; public class ByDateTimeResult @@ -12,18 +17,75 @@ public class ByNumericDimensionResult public int Value { get; set; } } -public class SingeSeriesDataset +/// +/// Serialise the derived types of IDataset +/// +/// +public class ResultTypeMappingConverter : JsonConverter where TType : IDataset +{ + [return: MaybeNull] + public override TType Read( + ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) => + throw new NotImplementedException(); + + public override void Write(Utf8JsonWriter writer, TType value, JsonSerializerOptions options) + { + if (value is MultiSeriesDatetimeDataset) + { + JsonSerializer.Serialize(writer, value as MultiSeriesDatetimeDataset, options); + } + else if (value is MultiSeriesDataset) + { + JsonSerializer.Serialize(writer, value as MultiSeriesDataset, options); + } + else if (value is SingleSeriesDataset) + { + JsonSerializer.Serialize(writer, value as SingleSeriesDataset, options); + } + else + { + throw new NotImplementedException(); + } + } +} + +// A marker interface to identify things we want to be able to return from the analytics API +public interface IDataset; + +public class SingleSeriesDataset : IDataset { public IDictionary Values { get; set; } = new Dictionary(); } -public class MultiSeriesDatetimeDataset(string name) +public class AuditHistory(AuditEntry auditEntry, string resourceType, string resourceId) +{ + public AuditEntry AuditEntry { get; set; } = auditEntry; + public string ResourceType { get; set; } = resourceType; + public string ResourceId { get; set; } = resourceId; +} + +public class EntityDataset(IEnumerable items) : IDataset +{ + public IEnumerable Items { get; set; } = items; +} + +public class MultiSeriesDatetimeDataset : IDataset +{ + public List Series { get; set; } = []; +} + +public class DatetimeSeries(string name) { public string Name { get; set; } = name; public List Periods { get; set; } = []; } -public class MultiSeriesDataset(string name, string dimension) +public class MultiSeriesDataset : IDataset +{ + public List Series { get; set; } = []; +} + +public class Series(string name, string dimension) { public string Name { get; set; } = name; public string Dimension { get; set; } = dimension; diff --git a/Btms.Backend.IntegrationTests/AnalyticsTests.cs b/Btms.Backend.IntegrationTests/AnalyticsTests.cs new file mode 100644 index 0000000..1cb9d98 --- /dev/null +++ b/Btms.Backend.IntegrationTests/AnalyticsTests.cs @@ -0,0 +1,105 @@ +using System.Diagnostics.CodeAnalysis; +using Btms.Common.Extensions; +using Btms.Model; +using Btms.SyncJob; +using Btms.Backend.IntegrationTests.JsonApiClient; +using FluentAssertions; +using System.Net; +using System.Net.Http.Json; +using System.Text; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Text.Json.Serialization; +using Btms.Backend.IntegrationTests.Extensions; +using Btms.Backend.IntegrationTests.Helpers; +using Json.More; +using Xunit; +using Xunit.Abstractions; + +namespace Btms.Backend.IntegrationTests; + +[Trait("Category", "Integration")] +public class AnalyticsTests(IntegrationTestsApplicationFactory factory, ITestOutputHelper testOutputHelper) + : BaseApiTests(factory, testOutputHelper), IClassFixture +{ + + // private static void ShouldNotBeNull([DoesNotReturnIf(true), NotNull] T? value) + // { + // // throw if null + // value.Should().NotBeNull(); + // } + + [Fact] + public async Task GetIndividualMultiSeriesDatetimeChart() + { + //Act + var response = await Client.GetAsync( + "/analytics/dashboard?chartsToRender=importNotificationLinkingByCreated"); + + // Assert + response.IsSuccessStatusCode.Should().BeTrue(response.StatusCode.ToString()); + + var responseDictionary = await response.ToJsonDictionary(); + + responseDictionary.Count.Should().Be(1); + + responseDictionary.Keys.Should().Equal("importNotificationLinkingByCreated"); + + } + + [Fact] + public async Task GetIndividualMultiSeriesChart() + { + //Act + var response = await Client.GetAsync( + "/analytics/dashboard?chartsToRender=lastMonthMovementsByItemCount"); + + // Assert + response.IsSuccessStatusCode.Should().BeTrue(response.StatusCode.ToString()); + + var responseDictionary = await response.ToJsonDictionary(); + + responseDictionary.Count.Should().Be(1); + + responseDictionary.Keys.Should().Equal("lastMonthMovementsByItemCount"); + + } + + [Fact] + public async Task GetIndividualSingleSeriesChart() + { + //Act + var response = await Client.GetAsync( + "/analytics/dashboard?chartsToRender=last7DaysImportNotificationsLinkingStatus"); + + // Assert + response.IsSuccessStatusCode.Should().BeTrue(response.StatusCode.ToString()); + + var responseDictionary = await response.ToJsonDictionary(); + + responseDictionary.Count.Should().Be(1); + + responseDictionary.Keys.Should().Equal("last7DaysImportNotificationsLinkingStatus"); + + } + + [Fact] + public async Task GetAllCharts() + { + //Act + var response = await Client.GetAsync("/analytics/dashboard"); + + // Assert + response.IsSuccessStatusCode.Should().BeTrue(response.StatusCode.ToString()); + + var responseDictionary = await response.ToJsonDictionary(); + + responseDictionary.Count.Should().Be(13); + + responseDictionary.Keys.Take(2).Should().Equal( + "importNotificationLinkingByCreated", + "importNotificationLinkingByArrival"); + + responseDictionary["importNotificationLinkingByCreated"].Should().NotBeNull(); + } +} \ No newline at end of file diff --git a/Btms.Backend.IntegrationTests/Extensions/HttpExtensions.cs b/Btms.Backend.IntegrationTests/Extensions/HttpExtensions.cs new file mode 100644 index 0000000..0c4ec61 --- /dev/null +++ b/Btms.Backend.IntegrationTests/Extensions/HttpExtensions.cs @@ -0,0 +1,14 @@ +using System.Text.Json.Nodes; +using Btms.Common.Extensions; + +namespace Btms.Backend.IntegrationTests.Extensions; + +public static class HttpExtensions +{ + public static async Task> ToJsonDictionary(this HttpResponseMessage? response) + { + var responseDictionary = (await response!.Content.ReadAsStringAsync()).ToJsonDictionary(); + ArgumentNullException.ThrowIfNull(responseDictionary); + return responseDictionary; + } +} \ No newline at end of file diff --git a/Btms.Backend/Config/AnalyticsDashboards.cs b/Btms.Backend/Config/AnalyticsDashboards.cs new file mode 100644 index 0000000..df25f6a --- /dev/null +++ b/Btms.Backend/Config/AnalyticsDashboards.cs @@ -0,0 +1,91 @@ +using Btms.Analytics; +using Btms.Analytics.Extensions; +using Btms.Common.Extensions; +using FluentAssertions; +using MongoDB.Driver.Linq; + +namespace Btms.Backend.Config; + +public static class AnalyticsDashboards +{ + public static async Task> GetCharts( + ILogger logger, + IImportNotificationsAggregationService importService, + IMovementsAggregationService movementsService, + string[] chartsToRender) + { + var charts = new Dictionary>> + { + { + "importNotificationLinkingByCreated", + () => importService.ByCreated(DateTime.Today.MonthAgo(), DateTime.Today).AsIDataset() + }, + { + "importNotificationLinkingByArrival", + () => importService.ByArrival(DateTime.Today.MonthAgo(), DateTime.Today.MonthLater()).AsIDataset() + }, + { + "last7DaysImportNotificationsLinkingStatus", + () => importService.ByStatus(DateTime.Today.WeekAgo(), DateTime.Now).AsIDataset() + }, + { + "last24HoursImportNotificationsLinkingStatus", + () => importService.ByStatus(DateTime.Now.Yesterday(), DateTime.Now).AsIDataset() + }, + { + "last24HoursImportNotificationsLinkingByCreated", + () => importService + .ByCreated(DateTime.Now.NextHour().Yesterday(), DateTime.Now.NextHour(), AggregationPeriod.Hour) + .AsIDataset() + }, + { + "lastMonthImportNotificationsByTypeAndStatus", + () => importService.ByStatus(DateTime.Today.MonthAgo(), DateTime.Now).AsIDataset() + }, + { + "last24HoursMovementsLinkingByCreated", + () => movementsService.ByCreated(DateTime.Now.NextHour().Yesterday(), DateTime.Now.NextHour(), AggregationPeriod.Hour).AsIDataset() + }, + { + "movementsLinkingByCreated", + () => movementsService.ByCreated(DateTime.Today.MonthAgo(), DateTime.Today).AsIDataset() + }, + { + "lastMonthMovementsByStatus", + () => movementsService.ByStatus(DateTime.Today.MonthAgo(), DateTime.Now).AsIDataset() + }, + { + "lastMonthMovementsByItemCount", + () => movementsService.ByItemCount(DateTime.Today.MonthAgo(), DateTime.Now).AsIDataset() + }, + { + "lastMonthMovementsByUniqueDocumentReferenceCount", + () => movementsService.ByUniqueDocumentReferenceCount(DateTime.Today.MonthAgo(), DateTime.Now).AsIDataset() + }, + { + "lastMonthUniqueDocumentReferenceByMovementCount", + () => movementsService.UniqueDocumentReferenceByMovementCount(DateTime.Today.MonthAgo(), DateTime.Now).AsIDataset() + }, + { + "lastMonthImportNotificationsByCommodityCount", + () => importService.ByCommodityCount(DateTime.Today.MonthAgo(), DateTime.Now).AsIDataset() + } + }; + + var chartsToReturn = chartsToRender.Length == 0 + ? charts + : charts.Where(keyValuePair => chartsToRender.Contains(keyValuePair.Key)); + + var taskList = chartsToReturn.Select(r => new KeyValuePair>(key:r.Key, value: r.Value())); + + await Task.WhenAll(taskList.Select(r => r.Value)); + + var output = taskList + .ToDictionary(t => t.Key, t => t.Value.Result); + + logger.LogInformation("Results found {0} Datasets, {1}", output.Count, output.Keys); + + return output; + + } +} \ No newline at end of file diff --git a/Btms.Backend/Endpoints/AnalyticsEndpoints.cs b/Btms.Backend/Endpoints/AnalyticsEndpoints.cs index cd3cb95..cfd7cfa 100644 --- a/Btms.Backend/Endpoints/AnalyticsEndpoints.cs +++ b/Btms.Backend/Endpoints/AnalyticsEndpoints.cs @@ -1,5 +1,9 @@ +using System.Text.Json; using Btms.Analytics; +using Btms.Backend.Config; +using Btms.Common; using Btms.Common.Extensions; +using Btms.Model.Extensions; using Microsoft.AspNetCore.Mvc; namespace Btms.Backend.Endpoints; @@ -12,7 +16,18 @@ public static void UseAnalyticsEndpoints(this IEndpointRouteBuilder app) { app.MapGet(BaseRoute + "/dashboard", GetDashboard).AllowAnonymous(); app.MapGet(BaseRoute + "/record-current-state", RecordCurrentState).AllowAnonymous(); + app.MapGet(BaseRoute + "/timeline", Timeline).AllowAnonymous(); } + private static async Task Timeline( + [FromServices] IImportNotificationsAggregationService importService, + [FromServices] IMovementsAggregationService movementsService, + [FromQuery] string movementId) + { + + + return TypedResults.Json(await movementsService.GetHistory(movementId)); + } + private static async Task RecordCurrentState( [FromServices] ImportNotificationMetrics importNotificationMetrics) { @@ -22,62 +37,22 @@ private static async Task RecordCurrentState( private static async Task GetDashboard( [FromServices] IImportNotificationsAggregationService importService, - [FromServices] IMovementsAggregationService movementsService) + [FromServices] IMovementsAggregationService movementsService, + [FromQuery] string[] chartsToRender) { - var importNotificationLinkingByCreated = await importService - .ByCreated(DateTime.Today.MonthAgo(), DateTime.Today); - - var importNotificationLinkingByArrival = await importService - .ByArrival(DateTime.Today.MonthAgo(), DateTime.Today.MonthLater()); - - var last7DaysImportNotificationsLinkingStatus = await importService - .ByStatus(DateTime.Today.WeekAgo(), DateTime.Now); - - var last24HoursImportNotificationsLinkingStatus = await importService - .ByStatus(DateTime.Now.Yesterday(), DateTime.Now); - - var last24HoursImportNotificationsLinkingByCreated = await importService - .ByCreated(DateTime.Now.NextHour().Yesterday(), DateTime.Now.NextHour(), AggregationPeriod.Hour); - - var lastMonthImportNotificationsByTypeAndStatus = await importService - .ByStatus(DateTime.Today.MonthAgo(), DateTime.Now); - - var last24HoursMovementsLinkingByCreated = await movementsService - .ByCreated(DateTime.Now.NextHour().Yesterday(), DateTime.Now.NextHour(), AggregationPeriod.Hour); - - var movementsLinkingByCreated = await movementsService - .ByCreated(DateTime.Today.MonthAgo(), DateTime.Today); - - var lastMonthMovementsByStatus = await movementsService - .ByStatus(DateTime.Today.MonthAgo(), DateTime.Now); - - var lastMonthMovementsByItemCount = await movementsService - .ByItemCount(DateTime.Today.MonthAgo(), DateTime.Now); - - var lastMonthMovementsByUniqueDocumentReferenceCount = await movementsService - .ByUniqueDocumentReferenceCount(DateTime.Today.MonthAgo(), DateTime.Now); - - var lastMonthUniqueDocumentReferenceByMovementCount = await movementsService - .UniqueDocumentReferenceByMovementCount(DateTime.Today.MonthAgo(), DateTime.Now); - - var lastMonthImportNotificationsByCommodityCount = await importService - .ByCommodityCount(DateTime.Today.MonthAgo(), DateTime.Now); - - return Results.Ok(new - { - importNotificationLinkingByCreated, - importNotificationLinkingByArrival, - last7DaysImportNotificationsLinkingStatus, - last24HoursImportNotificationsLinkingStatus, - last24HoursMovementsLinkingByCreated, - last24HoursImportNotificationsLinkingByCreated, - movementsLinkingByCreated, - lastMonthMovementsByStatus, - lastMonthMovementsByItemCount, - lastMonthImportNotificationsByCommodityCount, - lastMonthMovementsByUniqueDocumentReferenceCount, - lastMonthImportNotificationsByTypeAndStatus, - lastMonthUniqueDocumentReferenceByMovementCount - }); + var logger = ApplicationLogging.CreateLogger("AnalyticsEndpoints"); + var result = await AnalyticsDashboards.GetCharts(logger, importService, movementsService, chartsToRender); + + var options = + new JsonSerializerOptions + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + Converters = + { + new ResultTypeMappingConverter() + } + }; + + return TypedResults.Json(result, options); } } \ No newline at end of file diff --git a/Btms.Backend/Endpoints/SyncEndpoints.cs b/Btms.Backend/Endpoints/SyncEndpoints.cs index 56ef602..90727a2 100644 --- a/Btms.Backend/Endpoints/SyncEndpoints.cs +++ b/Btms.Backend/Endpoints/SyncEndpoints.cs @@ -27,6 +27,7 @@ public static void UseSyncEndpoints(this IEndpointRouteBuilder app, IOptions SyncGmrs([FromServices] IBtmsMediator mediato return Results.Accepted($"/sync/jobs/{command.JobId}", command.JobId); } - ////private static async Task GetSyncDecisions( - //// [FromServices] IBtmsMediator mediator, - //// SyncPeriod syncPeriod) - ////{ - //// SyncDecisionsCommand command = new() { SyncPeriod = syncPeriod }; - //// return await SyncDecisions(mediator, command); - ////} + private static async Task GetSyncDecisions( + [FromServices] IBtmsMediator mediator, + SyncPeriod syncPeriod) + { + SyncDecisionsCommand command = new() { SyncPeriod = syncPeriod }; + return await SyncDecisions(mediator, command); + } private static async Task SyncDecisions([FromServices] IBtmsMediator mediator, [FromBody] SyncDecisionsCommand command) diff --git a/Btms.Backend/Program.cs b/Btms.Backend/Program.cs index ab96634..b1d0663 100644 --- a/Btms.Backend/Program.cs +++ b/Btms.Backend/Program.cs @@ -38,6 +38,7 @@ using Btms.Azure.Extensions; using Environment = System.Environment; using Btms.Backend.Asb; +using Btms.Common; //-------- Configure the WebApplication builder------------------// @@ -229,6 +230,9 @@ static WebApplication BuildWebApplication(WebApplicationBuilder builder) { var app = builder.Build(); + // Allows us to make a global logger factory available for use where we can't get it from DI, e.g. from static functions + ApplicationLogging.LoggerFactory = app.Services.GetService(); + app.UseEmfExporter(); app.UseAuthentication(); app.UseAuthorization(); diff --git a/Btms.Common/Extensions/StringExtensions.cs b/Btms.Common/Extensions/StringExtensions.cs new file mode 100644 index 0000000..ca9e062 --- /dev/null +++ b/Btms.Common/Extensions/StringExtensions.cs @@ -0,0 +1,20 @@ +using System.Text.Json; +using System.Text.Json.Nodes; + +namespace Btms.Common.Extensions; + +public static class StringExtensions +{ + public static JsonNode? ToJsonNode(this string value) + { + return JsonNode.Parse(value); + } + public static JsonObject? ToJsonObject(this string value) + { + return JsonNode.Parse(value) as JsonObject; + } + public static IDictionary? ToJsonDictionary(this string value) + { + return JsonNode.Parse(value) as IDictionary; + } +} \ No newline at end of file diff --git a/TestDataGenerator/README.md b/TestDataGenerator/README.md index 114c115..66411f6 100644 --- a/TestDataGenerator/README.md +++ b/TestDataGenerator/README.md @@ -13,15 +13,15 @@ isn't quite working, so I've been generating locally and then syncing to blob st az account set --subscription 7d775166-9d6c-4ac2-91a5-61904bae5caa -az storage blob directory delete --container-name dmp-data-1001 --directory-path GENERATED-LOADTEST --account-name -snddmpinfdl1001 --recursive -az storage blob directory delete --container-name dmp-data-1001 --directory-path GENERATED-LOADTEST-BASIC --account-name -snddmpinfdl1001 --recursive +az storage blob directory delete --container-name dmp-data-1001 --directory-path GENERATED-LOADTEST --account-name snddmpinfdl1001 --recursive +az storage blob directory delete --container-name dmp-data-1001 --directory-path GENERATED-LOADTEST-BASIC --account-name snddmpinfdl1001 --recursive +az storage blob directory delete --container-name dmp-data-1001 --directory-path PRODREDACTED-20241204 --account-name snddmpinfdl1001 --recursive az storage blob directory upload -c dmp-data-1001 -d --account-name snddmpinfdl1001 -s TestDataGenerator/.test-data-generator/GENERATED-LOADTEST --recursive az storage blob upload-batch -d dmp-data-1001 --account-name snddmpinfdl1001 -s TestDataGenerator/.test-data-generator/GENERATED-LOADTEST-90Dx10k --destination-path GENERATED-LOADTEST-90Dx10k -~~~~az storage blob upload-batch -d dmp-data-1001 --account-name snddmpinfdl1001 -s -TestDataGenerator/.test-data-generator/GENERATED-LOADTEST-BASIC --destination-path GENERATED-LOADTEST-BASIC \ No newline at end of file + +az storage blob upload-batch -d dmp-data-1001 --account-name snddmpinfdl1001 -s TestDataGenerator/.test-data-generator/GENERATED-LOADTEST-BASIC --destination-path GENERATED-LOADTEST-BASIC +az storage blob upload-batch -d dmp-data-1001 --account-name snddmpinfdl1001 -s TestDataGenerator/.test-data-generator/PRODREDACTED-20241204 --destination-path PRODREDACTED-20241204 \ No newline at end of file