From 5259079d3ad154cc7a2ca9d0f17aa8e4e6f0e99a Mon Sep 17 00:00:00 2001 From: Craig Edmunds Date: Wed, 18 Dec 2024 15:29:22 +0000 Subject: [PATCH] CDMS-200 add test for movement not found (#23) * CDMS-200 add test for movement not found CDMS-200 improved CachingBlobService implementation * CMDS-200 fixes tests * CDMS-200 refactors message processing * removes new test * Add decions in test generator and push to blob storage * Tidies up audit entry timings and source systems * Fixes up tests * Adds context to decisions * make audit entry version nullable * Handles checks in clearance request builder * Try to send clearance requests with the DecisionNumber set to the decision consumer * Updated TestDataGeneratorHelpers to use the bus to route to the consumers rather than call directly * tidy up the switch statement further * CDMS-200 adds ability to control sequencing of messages to builders * Switch Decisions to their own type * Switched back to using consumers directly * Finishes initial decision metrics * Tidied up code errors * Fixed up tests --------- Co-authored-by: Thomas Anderson --- .../Btms.Analytics.Tests.csproj | 1 + .../Fixtures/BasicSampleDataTestFixture.cs | 46 +++++---- .../Fixtures/MultiItemDataTestFixture.cs | 45 ++++++--- .../Helpers/ITestOutputHelperExtensions.cs | 16 ++++ .../Helpers/TestDataGeneratorHelpers.cs | 94 +++++++++++++------ .../ImportNotificationsByArrivalDateTests.cs | 2 +- .../ImportNotificationsByCommoditiesTests.cs | 2 +- .../ImportNotificationsByCreatedDateTests.cs | 6 +- .../ImportNotificationsByStatusTests.cs | 6 +- Btms.Analytics.Tests/MovementHistoryTests.cs | 27 +++++- .../MovementsByCreatedDateTests.cs | 6 +- .../MovementsByDecisionsTests.cs | 32 +++++++ Btms.Analytics.Tests/MovementsByItemsTests.cs | 2 +- .../MovementsByStatusTests.cs | 6 +- ...MovementsByUniqueDocumentReferenceTests.cs | 2 +- ...ementsDocumentReferencesByMovementTests.cs | 2 +- .../IMovementsAggregationService.cs | 3 +- Btms.Analytics/MovementsAggregationService.cs | 45 ++++++++- Btms.Backend.Data/IMongoCollectionSet.cs | 3 + .../InMemory/MemoryCollectionSet.cs | 5 + Btms.Backend.Data/Mongo/MongoCollectionSet.cs | 6 ++ .../AnalyticsTests.cs | 2 - Btms.Backend.Test/Auditing/AuditingTests.cs | 4 +- Btms.Backend/Config/AnalyticsDashboards.cs | 4 + Btms.Backend/Endpoints/AnalyticsEndpoints.cs | 9 +- Btms.Backend/Endpoints/ManagementEndpoints.cs | 3 +- Btms.BlobService/CachingBlobService.cs | 55 +++++++---- .../Commands/SyncDecisionsCommandTests.cs | 2 +- .../Decisions/NoMatchDecisionsTest.cs | 7 +- .../Services/Matching/MatchingServiceTests.cs | 39 +++++--- .../Commands/SyncDecisionsCommand.cs | 2 +- .../PreProcessing/MovementPreProcessor.cs | 6 +- .../Decisions/DecisionMessageBuilder.cs | 12 +-- Btms.Common/Extensions/DateTimeExtensions.cs | 5 +- .../ClearanceRequestConsumerTests.cs | 4 +- .../NotificationsConsumerTests.cs | 4 +- .../AlvsClearanceRequestConsumer.cs | 6 +- Btms.Consumers/ConsumerOptions.cs | 1 + Btms.Consumers/DecisionsConsumer.cs | 6 +- .../Extensions/ServiceCollectionExtensions.cs | 6 +- Btms.Consumers/GmrConsumer.cs | 5 +- Btms.Consumers/NotificationConsumer.cs | 6 +- Btms.Model/Auditing/AuditEntry.cs | 59 +++++++----- Btms.Model/Ipaffs/ImportNotification.cs | 11 ++- Btms.Model/Movement.cs | 16 +++- .../Extensions/ServiceProviderExtensions.cs | 24 +++++ Btms.Tests.Common/Btms.Tests.Common.csproj | 18 ++++ Btms.Tests.Common/Class1.cs | 5 + .../DecisionMapper.g.cs | 32 +++++++ .../ClearanceRequestExtensions.cs | 2 +- Btms.Types.Alvs.V1/Decision.cs | 42 +++++++++ .../ClearanceRequestBuilderTests.cs | 2 +- .../ImportNotificationBuilderTests.cs | 2 +- TestDataGenerator/ClearanceRequestBuilder.cs | 17 +++- TestDataGenerator/DecisionBuilder.cs | 87 +++++++++++++++++ TestDataGenerator/Generator.cs | 21 +++-- TestDataGenerator/Helpers/DataHelpers.cs | 21 ++++- .../ImportNotificationBuilder.cs | 9 +- TestDataGenerator/Program.cs | 6 +- .../Properties/launchSettings.json | 11 +++ TestDataGenerator/README.md | 10 +- TestDataGenerator/ScenarioGenerator.cs | 64 ++++++++++++- .../Scenarios/CRNoMatchScenarioGenerator.cs | 4 +- .../ChedAManyCommoditiesScenarioGenerator.cs | 6 +- .../ChedANoMatchScenarioGenerator.cs | 4 +- .../ChedASimpleMatchScenarioGenerator.cs | 10 +- .../ChedPSimpleMatchScenarioGenerator.cs | 16 +++- .../Scenarios/Samples/cr-one-item.json | 2 +- .../Scenarios/Samples/decision-one-item.json | 29 ++++++ TestDataGenerator/TestDataGenerator.csproj | 8 +- 70 files changed, 857 insertions(+), 226 deletions(-) create mode 100644 Btms.Analytics.Tests/Helpers/ITestOutputHelperExtensions.cs create mode 100644 Btms.Analytics.Tests/MovementsByDecisionsTests.cs create mode 100644 Btms.SyncJob/Extensions/ServiceProviderExtensions.cs create mode 100644 Btms.Tests.Common/Btms.Tests.Common.csproj create mode 100644 Btms.Tests.Common/Class1.cs create mode 100644 Btms.Types.Alvs.Mapping.V1/DecisionMapper.g.cs create mode 100644 Btms.Types.Alvs.V1/Decision.cs create mode 100644 TestDataGenerator/DecisionBuilder.cs create mode 100644 TestDataGenerator/Scenarios/Samples/decision-one-item.json diff --git a/Btms.Analytics.Tests/Btms.Analytics.Tests.csproj b/Btms.Analytics.Tests/Btms.Analytics.Tests.csproj index a246796..b612975 100644 --- a/Btms.Analytics.Tests/Btms.Analytics.Tests.csproj +++ b/Btms.Analytics.Tests/Btms.Analytics.Tests.csproj @@ -20,6 +20,7 @@ + diff --git a/Btms.Analytics.Tests/Fixtures/BasicSampleDataTestFixture.cs b/Btms.Analytics.Tests/Fixtures/BasicSampleDataTestFixture.cs index 5ee8684..312e43d 100644 --- a/Btms.Analytics.Tests/Fixtures/BasicSampleDataTestFixture.cs +++ b/Btms.Analytics.Tests/Fixtures/BasicSampleDataTestFixture.cs @@ -2,7 +2,9 @@ using Btms.Backend.Data; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; using TestDataGenerator.Scenarios; +using Xunit.Abstractions; namespace Btms.Analytics.Tests.Fixtures; @@ -11,53 +13,65 @@ public class BasicSampleDataTestFixture : IDisposable #pragma warning restore S3881 { public IHost App; - public IImportNotificationsAggregationService ImportNotificationsAggregationService; - public IMovementsAggregationService MovementsAggregationService; - - public IMongoDbContext MongoDbContext; - public BasicSampleDataTestFixture() + + private readonly IMongoDbContext _mongoDbContext; + private readonly ILogger _logger; + public BasicSampleDataTestFixture(IMessageSink messageSink) { + _logger = messageSink.ToLogger(); + var builder = TestContextHelper.CreateBuilder(); App = builder.Build(); var rootScope = App.Services.CreateScope(); - MongoDbContext = rootScope.ServiceProvider.GetRequiredService(); - ImportNotificationsAggregationService = rootScope.ServiceProvider.GetRequiredService(); - MovementsAggregationService = rootScope.ServiceProvider.GetRequiredService(); + _mongoDbContext = rootScope.ServiceProvider.GetRequiredService(); // Would like to pick this up from env/config/DB state var insertToMongo = true; if (insertToMongo) { - MongoDbContext.ResetCollections().GetAwaiter().GetResult(); + _mongoDbContext.ResetCollections().GetAwaiter().GetResult(); // Ensure we have some data scenarios around 24/48 hour tests - App.PushToConsumers(App.CreateScenarioConfig(10, 3, arrivalDateRange: 0)) + App.PushToConsumers(_logger, App.CreateScenarioConfig(10, 3, arrivalDateRange: 0)) .GetAwaiter().GetResult(); - App.PushToConsumers(App.CreateScenarioConfig(10, 3, arrivalDateRange: 2)) + App.PushToConsumers(_logger, App.CreateScenarioConfig(10, 3, arrivalDateRange: 2)) .GetAwaiter().GetResult(); - App.PushToConsumers(App.CreateScenarioConfig(10, 3, arrivalDateRange: 0)) + App.PushToConsumers(_logger, App.CreateScenarioConfig(10, 3, arrivalDateRange: 0)) .GetAwaiter().GetResult(); // Create some more variable data over the rest of time - App.PushToConsumers( + App.PushToConsumers(_logger, App.CreateScenarioConfig(10, 7, arrivalDateRange: 10)) .GetAwaiter().GetResult(); - App.PushToConsumers(App.CreateScenarioConfig(5, 3, arrivalDateRange: 10)) + App.PushToConsumers(_logger, App.CreateScenarioConfig(5, 3, arrivalDateRange: 10)) .GetAwaiter().GetResult(); - App.PushToConsumers(App.CreateScenarioConfig(1, 3, arrivalDateRange: 10)) + App.PushToConsumers(_logger, App.CreateScenarioConfig(1, 3, arrivalDateRange: 10)) .GetAwaiter().GetResult(); - App.PushToConsumers(App.CreateScenarioConfig(1, 3, arrivalDateRange: 10)) + App.PushToConsumers(_logger, App.CreateScenarioConfig(1, 3, arrivalDateRange: 10)) .GetAwaiter().GetResult(); } } + + + public IImportNotificationsAggregationService GetImportNotificationsAggregationService(ITestOutputHelper testOutputHelper) + { + var logger = testOutputHelper.GetLogger(); + return new ImportNotificationsAggregationService(_mongoDbContext, logger); + } + + public IMovementsAggregationService GetMovementsAggregationService(ITestOutputHelper testOutputHelper) + { + var logger = testOutputHelper.GetLogger(); + return new MovementsAggregationService(_mongoDbContext, logger); + } public void Dispose() { diff --git a/Btms.Analytics.Tests/Fixtures/MultiItemDataTestFixture.cs b/Btms.Analytics.Tests/Fixtures/MultiItemDataTestFixture.cs index c322101..f39295c 100644 --- a/Btms.Analytics.Tests/Fixtures/MultiItemDataTestFixture.cs +++ b/Btms.Analytics.Tests/Fixtures/MultiItemDataTestFixture.cs @@ -1,7 +1,9 @@ using Btms.Analytics.Tests.Helpers; using Btms.Backend.Data; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; using TestDataGenerator.Scenarios; +using Xunit.Abstractions; namespace Btms.Analytics.Tests.Fixtures; @@ -9,20 +11,20 @@ namespace Btms.Analytics.Tests.Fixtures; public class MultiItemDataTestFixture : IDisposable #pragma warning restore S3881 { - public readonly IImportNotificationsAggregationService ImportNotificationsAggregationService; - public readonly IMovementsAggregationService MovementsAggregationService; + public readonly IMongoDbContext MongoDbContext; + private readonly IServiceScope _rootScope; + private readonly ILogger _logger; - public IMongoDbContext MongoDbContext; - public MultiItemDataTestFixture() + public MultiItemDataTestFixture(IMessageSink messageSink) { + _logger = messageSink.ToLogger(); + var builder = TestContextHelper.CreateBuilder(); var app = builder.Build(); - var rootScope = app.Services.CreateScope(); + _rootScope = app.Services.CreateScope(); - MongoDbContext = rootScope.ServiceProvider.GetRequiredService(); - ImportNotificationsAggregationService = rootScope.ServiceProvider.GetRequiredService(); - MovementsAggregationService = rootScope.ServiceProvider.GetRequiredService(); + MongoDbContext = _rootScope.ServiceProvider.GetRequiredService(); // Would like to pick this up from env/config/DB state var insertToMongo = true; @@ -31,16 +33,33 @@ public MultiItemDataTestFixture() { MongoDbContext.ResetCollections().GetAwaiter().GetResult(); - app.PushToConsumers(app.CreateScenarioConfig(10, 3, arrivalDateRange: 0)) + app.PushToConsumers(_logger, app.CreateScenarioConfig(10, 3, arrivalDateRange: 0)) .GetAwaiter().GetResult(); - - app.PushToConsumers(app.CreateScenarioConfig(10, 3, arrivalDateRange: 0)) + + app.PushToConsumers(_logger, app.CreateScenarioConfig(10, 3, arrivalDateRange: 0)) .GetAwaiter().GetResult(); - - app.PushToConsumers(app.CreateScenarioConfig(10, 3, arrivalDateRange: 0)) + + app.PushToConsumers(_logger, app.CreateScenarioConfig(10, 3, arrivalDateRange: 0)) + .GetAwaiter().GetResult(); + + app.PushToConsumers(_logger, app.CreateScenarioConfig(1, 1, arrivalDateRange: 0)) .GetAwaiter().GetResult(); } } + + public IImportNotificationsAggregationService GetImportNotificationsAggregationService(ITestOutputHelper testOutputHelper) + { + var logger = testOutputHelper.GetLogger(); + return new ImportNotificationsAggregationService(MongoDbContext, logger); + } + + public IMovementsAggregationService GetMovementsAggregationService(ITestOutputHelper testOutputHelper) + { + var logger = testOutputHelper.GetLogger(); + // return _rootScope.ServiceProvider.GetRequiredService(); + return new MovementsAggregationService(MongoDbContext, logger); + } + public void Dispose() { diff --git a/Btms.Analytics.Tests/Helpers/ITestOutputHelperExtensions.cs b/Btms.Analytics.Tests/Helpers/ITestOutputHelperExtensions.cs new file mode 100644 index 0000000..e495e10 --- /dev/null +++ b/Btms.Analytics.Tests/Helpers/ITestOutputHelperExtensions.cs @@ -0,0 +1,16 @@ +using MartinCostello.Logging.XUnit; +using Microsoft.Extensions.Logging; +using Xunit.Abstractions; + +namespace Btms.Analytics.Tests.Helpers; + +public static class ITestOutputHelperExtensions +{ + public static ILogger GetLogger(this ITestOutputHelper helper) + { + var loggerProvider = new XUnitLoggerProvider(helper, new XUnitLoggerOptions()); + var factory = new LoggerFactory([loggerProvider]); + + return factory.CreateLogger(); + } +} \ No newline at end of file diff --git a/Btms.Analytics.Tests/Helpers/TestDataGeneratorHelpers.cs b/Btms.Analytics.Tests/Helpers/TestDataGeneratorHelpers.cs index d7343f2..aaba184 100644 --- a/Btms.Analytics.Tests/Helpers/TestDataGeneratorHelpers.cs +++ b/Btms.Analytics.Tests/Helpers/TestDataGeneratorHelpers.cs @@ -1,6 +1,8 @@ +using Btms.Common.Extensions; using Btms.Consumers; using Btms.Types.Alvs; using Btms.Types.Ipaffs; +using Btms.SyncJob.Extensions; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; @@ -8,55 +10,89 @@ using SlimMessageBus.Host; using TestDataGenerator; using TestDataGenerator.Scenarios; +using Decision = Btms.Types.Alvs.Decision; namespace Btms.Analytics.Tests.Helpers; public static class TestDataGeneratorHelpers { - private static int _scenarioIndex; + private static int scenarioIndex; - public static async Task PushToConsumers(this IHost app, ScenarioConfig scenario) + public static async Task PushToConsumers(this IHost app, ILogger logger, ScenarioConfig scenario) { - var generatorResults = app.Generate(scenario); - _scenarioIndex++; + var generatorResults = app.Generate(logger, scenario); + scenarioIndex++; - app.Services.GetRequiredService>(); + // var logger = app.Services.GetRequiredService>(); + var bus = app.Services.GetRequiredService(); foreach (var generatorResult in generatorResults) { - foreach (var cr in generatorResult.ClearanceRequests) + foreach (var message in generatorResult) { var scope = app.Services.CreateScope(); - var consumer = (AlvsClearanceRequestConsumer)scope.ServiceProvider.GetRequiredService>(); - - consumer.Context = new ConsumerContext + + switch (message) { - Headers = new Dictionary { { "messageId", cr.Header!.EntryReference! } } - }; - - await consumer.OnHandle(cr); - } - - foreach (var n in generatorResult.ImportNotifications) - { - var scope = app.Services.CreateScope(); - var consumer = (NotificationConsumer)scope.ServiceProvider.GetRequiredService>(); - - consumer.Context = new ConsumerContext - { - Headers = new Dictionary { { "messageId", n.ReferenceNumber! } } - }; - - await consumer.OnHandle(n); + case null: + throw new ArgumentNullException(); + + case ImportNotification n: + + var notificationConsumer = (NotificationConsumer)scope + .ServiceProvider + .GetRequiredService>(); + + notificationConsumer.Context = new ConsumerContext + { + Headers = new Dictionary { { "messageId", n.ReferenceNumber! } } + }; + + await notificationConsumer.OnHandle(n); + logger.LogInformation("Sent notification {0} to consumer", n.ReferenceNumber!); + break; + + case Decision d: + + var decisionConsumer = (DecisionsConsumer)scope + .ServiceProvider + .GetRequiredService>(); + + decisionConsumer.Context = new ConsumerContext + { + Headers = new Dictionary { { "messageId", d.Header!.EntryReference! } } + }; + + await decisionConsumer.OnHandle(d); + logger.LogInformation("Sent decision {0} to consumer", d.Header!.EntryReference!); + break; + + case AlvsClearanceRequest cr: + + var crConsumer = (AlvsClearanceRequestConsumer)scope + .ServiceProvider + .GetRequiredService>(); + + crConsumer.Context = new ConsumerContext + { + Headers = new Dictionary { { "messageId", cr.Header!.EntryReference! } } + }; + + await crConsumer.OnHandle(cr); + logger.LogInformation("Semt cr {0} to consumer", cr.Header!.EntryReference!); + break; + + default: + throw new ArgumentException($"Unexpected type {message.GetType().Name}"); + } } } return app; } - private static ScenarioGenerator.GeneratorResult[] Generate(this IHost app, ScenarioConfig scenario) + private static ScenarioGenerator.GeneratorResult[] Generate(this IHost app, ILogger logger, ScenarioConfig scenario) { - var logger = app.Services.GetRequiredService>(); var days = scenario.CreationDateRange; var count = scenario.Count; var generator = scenario.Generator; @@ -73,7 +109,7 @@ private static ScenarioGenerator.GeneratorResult[] Generate(this IHost app, Scen { logger.LogInformation("Generating item {I}", i); - results.Add(generator.Generate(_scenarioIndex, i, entryDate, scenario)); + results.Add(generator.Generate(scenarioIndex, i, entryDate, scenario)); } } diff --git a/Btms.Analytics.Tests/ImportNotificationsByArrivalDateTests.cs b/Btms.Analytics.Tests/ImportNotificationsByArrivalDateTests.cs index 61c9baf..9034a8f 100644 --- a/Btms.Analytics.Tests/ImportNotificationsByArrivalDateTests.cs +++ b/Btms.Analytics.Tests/ImportNotificationsByArrivalDateTests.cs @@ -18,7 +18,7 @@ public async Task WhenCalledNextMonth_ReturnExpectedAggregation() { testOutputHelper.WriteLine("Querying for aggregated data"); - var result = (await basicSampleDataTestFixture.ImportNotificationsAggregationService + var result = (await basicSampleDataTestFixture.GetImportNotificationsAggregationService(testOutputHelper) .ByArrival(DateTime.Today, DateTime.Today.MonthLater())) .Series .ToList(); diff --git a/Btms.Analytics.Tests/ImportNotificationsByCommoditiesTests.cs b/Btms.Analytics.Tests/ImportNotificationsByCommoditiesTests.cs index 10ce1dc..735ba3d 100644 --- a/Btms.Analytics.Tests/ImportNotificationsByCommoditiesTests.cs +++ b/Btms.Analytics.Tests/ImportNotificationsByCommoditiesTests.cs @@ -18,7 +18,7 @@ public class ImportNotificationsByCommoditiesTests( public async Task WhenCalledLastWeek_ReturnExpectedAggregation() { testOutputHelper.WriteLine("Querying for aggregated data"); - var result = (await multiItemDataTestFixture.ImportNotificationsAggregationService + var result = (await multiItemDataTestFixture.GetImportNotificationsAggregationService(testOutputHelper) .ByCommodityCount(DateTime.Today.WeekAgo(), DateTime.Today.Tomorrow())) .Series .ToList(); diff --git a/Btms.Analytics.Tests/ImportNotificationsByCreatedDateTests.cs b/Btms.Analytics.Tests/ImportNotificationsByCreatedDateTests.cs index de4e3f1..a3ba540 100644 --- a/Btms.Analytics.Tests/ImportNotificationsByCreatedDateTests.cs +++ b/Btms.Analytics.Tests/ImportNotificationsByCreatedDateTests.cs @@ -16,7 +16,7 @@ public class ImportNotificationsByCreatedDateTests( [Fact] public async Task WhenCalledLast48Hours_ReturnExpectedAggregation() { - var result = (await basicSampleDataTestFixture.ImportNotificationsAggregationService + var result = (await basicSampleDataTestFixture.GetImportNotificationsAggregationService(testOutputHelper) .ByCreated(DateTime.Now.NextHour().AddDays(-2), DateTime.Now.NextHour(),AggregationPeriod.Hour)) .Series .ToList(); @@ -35,7 +35,7 @@ public async Task WhenCalledLast48Hours_ReturnExpectedAggregation() [Fact] public async Task WhenCalledLastMonth_ReturnExpectedAggregation() { - var result = (await basicSampleDataTestFixture.ImportNotificationsAggregationService + var result = (await basicSampleDataTestFixture.GetImportNotificationsAggregationService(testOutputHelper) .ByCreated(DateTime.Today.MonthAgo(), DateTime.Today.Tomorrow())) .Series .ToList(); @@ -63,7 +63,7 @@ public async Task WhenCalledWithTimePeriodYieldingNoResults_ReturnEmptyAggregati var from = DateTime.MaxValue.AddDays(-1); var to = DateTime.MaxValue; - var result = (await basicSampleDataTestFixture.ImportNotificationsAggregationService + var result = (await basicSampleDataTestFixture.GetImportNotificationsAggregationService(testOutputHelper) .ByCreated(from, to, AggregationPeriod.Hour)) .Series .ToList(); diff --git a/Btms.Analytics.Tests/ImportNotificationsByStatusTests.cs b/Btms.Analytics.Tests/ImportNotificationsByStatusTests.cs index 8ad2745..07e3063 100644 --- a/Btms.Analytics.Tests/ImportNotificationsByStatusTests.cs +++ b/Btms.Analytics.Tests/ImportNotificationsByStatusTests.cs @@ -17,7 +17,7 @@ public class ImportNotificationsByStatusTests( public async Task WhenCalledLastWeek_ReturnExpectedAggregation() { testOutputHelper.WriteLine("Querying for aggregated data"); - var result = (await basicSampleDataTestFixture.ImportNotificationsAggregationService + var result = (await basicSampleDataTestFixture.GetImportNotificationsAggregationService(testOutputHelper) .ByStatus(DateTime.Today.WeekAgo(), DateTime.Today.Tomorrow())); testOutputHelper.WriteLine("{0} aggregated items found", result.Values.Count); @@ -29,7 +29,7 @@ public async Task WhenCalledLastWeek_ReturnExpectedAggregation() public async Task WhenCalledLast48Hours_ReturnExpectedAggregation() { testOutputHelper.WriteLine("Querying for aggregated data"); - var result = (await basicSampleDataTestFixture.ImportNotificationsAggregationService + var result = (await basicSampleDataTestFixture.GetImportNotificationsAggregationService(testOutputHelper) .ByStatus(DateTime.Now.NextHour().AddDays(-2), DateTime.Now.NextHour())); testOutputHelper.WriteLine($"{result.Values.Count} aggregated items found"); @@ -44,7 +44,7 @@ public async Task WhenCalledLast48Hours_ReturnExpectedAggregation() public async Task WhenCalledWithTimePeriodYieldingNoResults_ReturnEmptyAggregation() { testOutputHelper.WriteLine("Querying for aggregated data"); - var result = (await basicSampleDataTestFixture.ImportNotificationsAggregationService + var result = (await basicSampleDataTestFixture.GetImportNotificationsAggregationService(testOutputHelper) .ByStatus(DateTime.MaxValue.AddDays(-1), DateTime.MaxValue)); testOutputHelper.WriteLine($"{result.Values.Count} aggregated items found"); diff --git a/Btms.Analytics.Tests/MovementHistoryTests.cs b/Btms.Analytics.Tests/MovementHistoryTests.cs index 8f356f7..d8d62e9 100644 --- a/Btms.Analytics.Tests/MovementHistoryTests.cs +++ b/Btms.Analytics.Tests/MovementHistoryTests.cs @@ -14,15 +14,34 @@ public class MovementHistoryTests( ITestOutputHelper testOutputHelper) { [Fact] - public async Task WhenCalled_ReturnsHistory() + public async Task WhenCalledWithAMovementThatHasADecision_ReturnsHistory() { + testOutputHelper.WriteLine("Find a suitable Movement with history"); + var movement = await multiItemDataTestFixture.MongoDbContext.Movements.Find( + m => + m.Relationships.Notifications.Data.Count > 0 + && m.Decisions.Count > 0); + + ArgumentNullException.ThrowIfNull(movement); + testOutputHelper.WriteLine("Querying for history"); - var result = await multiItemDataTestFixture.MovementsAggregationService - .GetHistory("23GB9999001215000001"); + var result = await multiItemDataTestFixture + .GetMovementsAggregationService(testOutputHelper) + .GetHistory(movement.Id!); - testOutputHelper.WriteLine("{0} history items found", result.Items.Count()); + testOutputHelper.WriteLine("{0} history items found", result!.Items.Count()); result.Items.Should().HasValue(); result.Items.Select(a => a.AuditEntry.CreatedSource).Should().BeInAscendingOrder(); } + + [Fact] + public async Task WhenCalledWithAFakeMovementID_ReturnsNoHistory() + { + testOutputHelper.WriteLine("Querying for history"); + var result = await multiItemDataTestFixture.GetMovementsAggregationService(testOutputHelper) + .GetHistory(""); + + result.Should().BeNull(); + } } \ No newline at end of file diff --git a/Btms.Analytics.Tests/MovementsByCreatedDateTests.cs b/Btms.Analytics.Tests/MovementsByCreatedDateTests.cs index e0173b3..c90975f 100644 --- a/Btms.Analytics.Tests/MovementsByCreatedDateTests.cs +++ b/Btms.Analytics.Tests/MovementsByCreatedDateTests.cs @@ -15,7 +15,7 @@ public class MovementsByCreatedDateTests( [Fact] public async Task WhenCalledLast48Hours_ReturnExpectedAggregation() { - var result = (await basicSampleDataTestFixture.MovementsAggregationService + var result = (await basicSampleDataTestFixture.GetMovementsAggregationService(testOutputHelper) .ByCreated(DateTime.Now.NextHour().AddDays(-2), DateTime.Now.NextHour(), AggregationPeriod.Hour)) .Series .ToList(); @@ -37,7 +37,7 @@ public async Task WhenCalledWithTimePeriodYieldingNoResults_ReturnEmptyAggregati var from = DateTime.MaxValue.AddDays(-1); var to = DateTime.MaxValue; - var result = (await basicSampleDataTestFixture.MovementsAggregationService + var result = (await basicSampleDataTestFixture.GetMovementsAggregationService(testOutputHelper) .ByCreated(from, to, AggregationPeriod.Hour)) .Series .ToList(); @@ -62,7 +62,7 @@ public async Task WhenCalledWithTimePeriodYieldingNoResults_ReturnEmptyAggregati [Fact] public async Task WhenCalledLastMonth_ReturnExpectedAggregation() { - var result = (await basicSampleDataTestFixture.MovementsAggregationService + var result = (await basicSampleDataTestFixture.GetMovementsAggregationService(testOutputHelper) .ByCreated(DateTime.Today.MonthAgo(), DateTime.Today.Tomorrow())) .Series .ToList(); diff --git a/Btms.Analytics.Tests/MovementsByDecisionsTests.cs b/Btms.Analytics.Tests/MovementsByDecisionsTests.cs new file mode 100644 index 0000000..ae215ee --- /dev/null +++ b/Btms.Analytics.Tests/MovementsByDecisionsTests.cs @@ -0,0 +1,32 @@ +using Btms.Common.Extensions; +using FluentAssertions; +using Xunit; +using Xunit.Abstractions; + +using Btms.Analytics.Tests.Fixtures; +using Btms.Analytics.Tests.Helpers; + +namespace Btms.Analytics.Tests; + +[Collection(nameof(MultiItemDataTestCollection))] +public class MovementsByDecisionsTests( + MultiItemDataTestFixture multiItemDataTestFixture, + ITestOutputHelper testOutputHelper) +{ + + [Fact] + public async Task WhenCalled_ReturnExpectedAggregation() + { + testOutputHelper.WriteLine("Querying for aggregated data"); + var result = (await multiItemDataTestFixture.GetMovementsAggregationService(testOutputHelper) + .ByDecision(DateTime.Today.MonthAgo(), DateTime.Today.Tomorrow())) + .Values + .ToList(); + + testOutputHelper.WriteLine("{0} aggregated items found", result.Count); + + result.Count.Should().Be(3); + result.Select(r => r.Key).Order().Should() + .Equal("ALVS Linked : H01", "BTMS Linked : X00", "BTMS Not Linked : X00"); + } +} \ No newline at end of file diff --git a/Btms.Analytics.Tests/MovementsByItemsTests.cs b/Btms.Analytics.Tests/MovementsByItemsTests.cs index 477367a..0d82f73 100644 --- a/Btms.Analytics.Tests/MovementsByItemsTests.cs +++ b/Btms.Analytics.Tests/MovementsByItemsTests.cs @@ -18,7 +18,7 @@ public class MovementsByItemsTests( public async Task WhenCalledLastWeek_ReturnExpectedAggregation() { testOutputHelper.WriteLine("Querying for aggregated data"); - var result = (await multiItemDataTestFixture.MovementsAggregationService + var result = (await multiItemDataTestFixture.GetMovementsAggregationService(testOutputHelper) .ByItemCount(DateTime.Today.WeekAgo(), DateTime.Today.Tomorrow())) .Series .ToList(); diff --git a/Btms.Analytics.Tests/MovementsByStatusTests.cs b/Btms.Analytics.Tests/MovementsByStatusTests.cs index 006ab2f..a66f0a4 100644 --- a/Btms.Analytics.Tests/MovementsByStatusTests.cs +++ b/Btms.Analytics.Tests/MovementsByStatusTests.cs @@ -16,7 +16,7 @@ public class MovementsByStatusTests( public async Task WhenCalledLastWeek_ReturnExpectedAggregation() { testOutputHelper.WriteLine("Querying for aggregated data"); - var result = (await basicSampleDataTestFixture.MovementsAggregationService + var result = (await basicSampleDataTestFixture.GetMovementsAggregationService(testOutputHelper) .ByStatus(DateTime.Today.WeekAgo(), DateTime.Today.Tomorrow())); testOutputHelper.WriteLine("{0} aggregated items found", result.Values.Count); @@ -29,7 +29,7 @@ public async Task WhenCalledLastWeek_ReturnExpectedAggregation() public async Task WhenCalledLast48Hours_ReturnExpectedAggregation() { testOutputHelper.WriteLine("Querying for aggregated data"); - var result = (await basicSampleDataTestFixture.MovementsAggregationService + var result = (await basicSampleDataTestFixture.GetMovementsAggregationService(testOutputHelper) .ByStatus(DateTime.Now.NextHour().AddDays(-2), DateTime.Now.NextHour())); testOutputHelper.WriteLine($"{result.Values.Count} aggregated items found"); @@ -42,7 +42,7 @@ public async Task WhenCalledLast48Hours_ReturnExpectedAggregation() public async Task WhenCalledWithTimePeriodYieldingNoResults_ReturnEmptyAggregation() { testOutputHelper.WriteLine("Querying for aggregated data"); - var result = (await basicSampleDataTestFixture.MovementsAggregationService + var result = (await basicSampleDataTestFixture.GetMovementsAggregationService(testOutputHelper) .ByStatus(DateTime.MaxValue.AddDays(-1), DateTime.MaxValue)); testOutputHelper.WriteLine($"{result.Values.Count} aggregated items found"); diff --git a/Btms.Analytics.Tests/MovementsByUniqueDocumentReferenceTests.cs b/Btms.Analytics.Tests/MovementsByUniqueDocumentReferenceTests.cs index 52c61f3..f7a17e2 100644 --- a/Btms.Analytics.Tests/MovementsByUniqueDocumentReferenceTests.cs +++ b/Btms.Analytics.Tests/MovementsByUniqueDocumentReferenceTests.cs @@ -18,7 +18,7 @@ public class MovementsByUniqueDocumentReferenceTests( public async Task WhenCalledLastWeek_ReturnExpectedAggregation() { testOutputHelper.WriteLine("Querying for aggregated data"); - var result = (await multiItemDataTestFixture.MovementsAggregationService + var result = (await multiItemDataTestFixture.GetMovementsAggregationService(testOutputHelper) .ByUniqueDocumentReferenceCount(DateTime.Today.WeekAgo(), DateTime.Today.Tomorrow())) .Series .ToList(); diff --git a/Btms.Analytics.Tests/MovementsDocumentReferencesByMovementTests.cs b/Btms.Analytics.Tests/MovementsDocumentReferencesByMovementTests.cs index 6a2ed05..4de74ec 100644 --- a/Btms.Analytics.Tests/MovementsDocumentReferencesByMovementTests.cs +++ b/Btms.Analytics.Tests/MovementsDocumentReferencesByMovementTests.cs @@ -16,7 +16,7 @@ public class MovementsDocumentReferencesByMovementTests( public async Task WhenCalledLastWeek_ReturnExpectedAggregation() { testOutputHelper.WriteLine("Querying for aggregated data"); - var result = (await multiItemDataTestFixture.MovementsAggregationService + var result = (await multiItemDataTestFixture.GetMovementsAggregationService(testOutputHelper) .UniqueDocumentReferenceByMovementCount(DateTime.Today.WeekAgo(), DateTime.Today.Tomorrow())); testOutputHelper.WriteLine("{0} aggregated items found", result.Values.Count); diff --git a/Btms.Analytics/IMovementsAggregationService.cs b/Btms.Analytics/IMovementsAggregationService.cs index cd65eda..948c8ec 100644 --- a/Btms.Analytics/IMovementsAggregationService.cs +++ b/Btms.Analytics/IMovementsAggregationService.cs @@ -7,8 +7,9 @@ public interface IMovementsAggregationService public Task ByCreated(DateTime from, DateTime to, AggregationPeriod aggregateBy = AggregationPeriod.Day); public Task ByStatus(DateTime from, DateTime to); public Task ByItemCount(DateTime from, DateTime to); + public Task ByDecision(DateTime from, DateTime to); public Task ByUniqueDocumentReferenceCount(DateTime from, DateTime to); public Task UniqueDocumentReferenceByMovementCount(DateTime from, DateTime to); public Task ByCheck(DateTime from, DateTime to); - public Task> GetHistory(string movementId); + public Task?> GetHistory(string movementId); } \ No newline at end of file diff --git a/Btms.Analytics/MovementsAggregationService.cs b/Btms.Analytics/MovementsAggregationService.cs index 7ce5169..9b14f6a 100644 --- a/Btms.Analytics/MovementsAggregationService.cs +++ b/Btms.Analytics/MovementsAggregationService.cs @@ -2,6 +2,7 @@ using System.Linq.Expressions; using Btms.Analytics.Extensions; using Btms.Backend.Data; +using Btms.Common.Extensions; using Btms.Model.Extensions; using Btms.Model; using Btms.Model.Auditing; @@ -148,12 +149,17 @@ public Task UniqueDocumentReferenceByMovementCount(DateTime return Task.FromResult(result); } - public async Task> GetHistory(string movementId) + public async Task?> GetHistory(string movementId) { var movement = await context .Movements .Find(movementId); + if (!movement.HasValue()) + { + return null; + } + var notificationIds = movement!.Relationships.Notifications.Data.Select(n => n.Id); var notificationEntries = context.Notifications @@ -202,4 +208,41 @@ private Task Aggregate(DateTime[] dateRange, Func ByDecision(DateTime from, DateTime to) + { + var mongoQuery = context + .Movements + .Where(m => m.CreatedSource >= from && m.CreatedSource < to) + .SelectMany(m => m.Decisions.Select(d => new { Decision = d, Movement = m })) + .SelectMany(m => + m.Decision.Items!.Select(i => new { Decision = m.Decision, Movement = m.Movement, Item = i })) + .SelectMany(m => m.Item.Checks!.Select(c => new + { + CheckCode = c.CheckCode, + DecisionCode = c.DecisionCode, + DecisionSourceSystem = m.Decision.ServiceHeader!.SourceSystem, + DecisionEntryReference = m.Decision.Header!.EntryReference, + DecisionEntryVersionNumber = m.Decision.Header!.EntryVersionNumber, + Movement = m.Movement.EntryReference, + MovementVersion = m.Movement.EntryVersionNumber, + HasLinks = m.Movement.Relationships.Notifications.Data.Count > 0, + ItemNumber = m.Item.ItemNumber + })) + .GroupBy(m => new { m.HasLinks, m.DecisionSourceSystem, m.DecisionCode }) + .Select(m => new { m.Key.HasLinks, m.Key.DecisionSourceSystem, m.Key.DecisionCode, Count = m.Count() }) + .ToList(); + + logger.LogInformation("Found {0} items", mongoQuery.Count); + logger.LogInformation(mongoQuery.ToJsonString()); + + return Task.FromResult(new SingleSeriesDataset() + { + Values = mongoQuery + .ToDictionary( + r => $"{ r.DecisionSourceSystem } { ( r.HasLinks ? "Linked" : "Not Linked" ) } : { r.DecisionCode }", + r => r.Count + ) + }); + } } \ No newline at end of file diff --git a/Btms.Backend.Data/IMongoCollectionSet.cs b/Btms.Backend.Data/IMongoCollectionSet.cs index 29a8967..164f8ab 100644 --- a/Btms.Backend.Data/IMongoCollectionSet.cs +++ b/Btms.Backend.Data/IMongoCollectionSet.cs @@ -1,3 +1,4 @@ +using System.Linq.Expressions; using Btms.Model.Data; using MongoDB.Driver; @@ -6,6 +7,8 @@ namespace Btms.Backend.Data; public interface IMongoCollectionSet : IQueryable where T : IDataEntity { Task Find(string id); + Task Find(Expression> query); + Task Insert(T item, IMongoDbTransaction transaction = default!, CancellationToken cancellationToken = default); Task Update(T item, string etag, IMongoDbTransaction transaction = default!, diff --git a/Btms.Backend.Data/InMemory/MemoryCollectionSet.cs b/Btms.Backend.Data/InMemory/MemoryCollectionSet.cs index 913c804..36295af 100644 --- a/Btms.Backend.Data/InMemory/MemoryCollectionSet.cs +++ b/Btms.Backend.Data/InMemory/MemoryCollectionSet.cs @@ -31,6 +31,11 @@ IEnumerator IEnumerable.GetEnumerator() return Task.FromResult(data.Find(x => x.Id == id)); } + public Task Find(Expression> query) + { + return Task.FromResult(data.Find(i => query.Compile()(i))); + } + public Task Insert(T item, IMongoDbTransaction transaction = default!, CancellationToken cancellationToken = default) { item._Etag = BsonObjectIdGenerator.Instance.GenerateId(null, null).ToString()!; diff --git a/Btms.Backend.Data/Mongo/MongoCollectionSet.cs b/Btms.Backend.Data/Mongo/MongoCollectionSet.cs index 6490ec2..a8954f7 100644 --- a/Btms.Backend.Data/Mongo/MongoCollectionSet.cs +++ b/Btms.Backend.Data/Mongo/MongoCollectionSet.cs @@ -4,6 +4,7 @@ using MongoDB.Driver.Linq; using System.Collections; using System.Linq.Expressions; +using Microsoft.EntityFrameworkCore; namespace Btms.Backend.Data.Mongo; @@ -35,6 +36,11 @@ IEnumerator IEnumerable.GetEnumerator() return await EntityQueryable.SingleOrDefaultAsync(x => x.Id == id); } + public async Task Find(Expression> query) + { + return await EntityQueryable.FirstOrDefaultAsync(query); + } + public Task Insert(T item, IMongoDbTransaction? transaction, CancellationToken cancellationToken = default) { item._Etag = BsonObjectIdGenerator.Instance.GenerateId(null, null).ToString()!; diff --git a/Btms.Backend.IntegrationTests/AnalyticsTests.cs b/Btms.Backend.IntegrationTests/AnalyticsTests.cs index 1cb9d98..598be38 100644 --- a/Btms.Backend.IntegrationTests/AnalyticsTests.cs +++ b/Btms.Backend.IntegrationTests/AnalyticsTests.cs @@ -94,8 +94,6 @@ public async Task GetAllCharts() var responseDictionary = await response.ToJsonDictionary(); - responseDictionary.Count.Should().Be(13); - responseDictionary.Keys.Take(2).Should().Equal( "importNotificationLinkingByCreated", "importNotificationLinkingByArrival"); diff --git a/Btms.Backend.Test/Auditing/AuditingTests.cs b/Btms.Backend.Test/Auditing/AuditingTests.cs index 42bc5c3..8d48281 100644 --- a/Btms.Backend.Test/Auditing/AuditingTests.cs +++ b/Btms.Backend.Test/Auditing/AuditingTests.cs @@ -15,7 +15,7 @@ public void CreateAuditWhenDifferentIsDouble() { var previous = new TestClassOne { NumberValue = 1.2 }; var current = new TestClassOne { NumberValue = 2.2 }; - var auditEntry = AuditEntry.CreateUpdated(previous, current, "testid", 1, DateTime.UtcNow); + var auditEntry = AuditEntry.CreateUpdated(previous, current, "testid", 1, DateTime.UtcNow, AuditEntry.CreatedByIpaffs); auditEntry.Should().NotBeNull(); auditEntry.Diff.Count.Should().Be(1); @@ -31,7 +31,7 @@ public void CreateAuditWhenDifferentIsInt() { var previous = new TestClassOne { NumberValue = 1 }; var current = new TestClassOne { NumberValue = 2 }; - var auditEntry = AuditEntry.CreateUpdated(previous, current, "testid", 1, DateTime.UtcNow); + var auditEntry = AuditEntry.CreateUpdated(previous, current, "testid", 1, DateTime.UtcNow,AuditEntry.CreatedByIpaffs); auditEntry.Should().NotBeNull(); auditEntry.Diff.Count.Should().Be(1); diff --git a/Btms.Backend/Config/AnalyticsDashboards.cs b/Btms.Backend/Config/AnalyticsDashboards.cs index df25f6a..ef4c1c0 100644 --- a/Btms.Backend/Config/AnalyticsDashboards.cs +++ b/Btms.Backend/Config/AnalyticsDashboards.cs @@ -69,6 +69,10 @@ public static async Task> GetCharts( { "lastMonthImportNotificationsByCommodityCount", () => importService.ByCommodityCount(DateTime.Today.MonthAgo(), DateTime.Now).AsIDataset() + }, + { + "lastMonthDecisionsByStatus", + () => movementsService.ByDecision(DateTime.Today.MonthAgo(), DateTime.Now).AsIDataset() } }; diff --git a/Btms.Backend/Endpoints/AnalyticsEndpoints.cs b/Btms.Backend/Endpoints/AnalyticsEndpoints.cs index cfd7cfa..ac42f21 100644 --- a/Btms.Backend/Endpoints/AnalyticsEndpoints.cs +++ b/Btms.Backend/Endpoints/AnalyticsEndpoints.cs @@ -16,16 +16,21 @@ public static void UseAnalyticsEndpoints(this IEndpointRouteBuilder app) { app.MapGet(BaseRoute + "/dashboard", GetDashboard).AllowAnonymous(); app.MapGet(BaseRoute + "/record-current-state", RecordCurrentState).AllowAnonymous(); - app.MapGet(BaseRoute + "/timeline", Timeline).AllowAnonymous(); + app.MapGet(BaseRoute + "/timeline", Timeline); } private static async Task Timeline( [FromServices] IImportNotificationsAggregationService importService, [FromServices] IMovementsAggregationService movementsService, [FromQuery] string movementId) { + var result = await movementsService.GetHistory(movementId); + if (result.HasValue()) + { + return TypedResults.Json(result); + } - return TypedResults.Json(await movementsService.GetHistory(movementId)); + return Results.NotFound(); } private static async Task RecordCurrentState( diff --git a/Btms.Backend/Endpoints/ManagementEndpoints.cs b/Btms.Backend/Endpoints/ManagementEndpoints.cs index 73d2dd3..61e4f59 100644 --- a/Btms.Backend/Endpoints/ManagementEndpoints.cs +++ b/Btms.Backend/Endpoints/ManagementEndpoints.cs @@ -36,7 +36,8 @@ private static bool RedactKeys(string key) return key.StartsWith("AZURE") || key.StartsWith("BlobServiceOptions__Azure") || key.StartsWith("ServiceBusOptions__ConnectionString") || - key.Contains("password", StringComparison.OrdinalIgnoreCase) || + key.StartsWith("AuthKeyStore__Credentials") || + key.Contains("password", StringComparison.OrdinalIgnoreCase) || _keysToRedact.Contains(key); } diff --git a/Btms.BlobService/CachingBlobService.cs b/Btms.BlobService/CachingBlobService.cs index 9019aa6..0a5026b 100644 --- a/Btms.BlobService/CachingBlobService.cs +++ b/Btms.BlobService/CachingBlobService.cs @@ -13,7 +13,7 @@ IOptions options { public Task CheckBlobAsync(int timeout = default, int retries = default) { - return blobService.CheckBlobAsync(); + return blobService.CheckBlobAsync(timeout, retries); } public Task CheckBlobAsync(string uri, int timeout = default, int retries = default) @@ -23,29 +23,52 @@ public Task CheckBlobAsync(string uri, int timeout = default, int retrie public async IAsyncEnumerable GetResourcesAsync(string prefix, [EnumeratorCancellation] CancellationToken cancellationToken) { - var path = Path.GetFullPath($"{options.Value.CachePath}/{prefix}"); - logger.LogInformation("Scanning disk {Path}", path); - - if (Directory.Exists(path)) + if (!options.Value.CacheReadEnabled) { - logger.LogInformation("Folder {Path} exists, looking for files", path); - foreach (var f in Directory.GetFiles(path, "*.json", SearchOption.AllDirectories)) - { - var relativePath = Path.GetRelativePath($"{Directory.GetCurrentDirectory()}/{options.Value.CachePath}", f); - logger.LogInformation("Found file {RelativePath}", relativePath); - yield return await Task.FromResult(new BtmsBlobItem { Name = relativePath }); - } + await foreach (var blobItem in blobService.GetResourcesAsync(prefix, cancellationToken)) + { + yield return blobItem; + } } - else{ - logger.LogWarning("Cannot scan folder {Path} as it doesn't exist", path); + else + { + var path = Path.GetFullPath($"{options.Value.CachePath}/{prefix}"); + logger.LogInformation("Scanning disk {Path}", path); + + if (Directory.Exists(path)) + { + logger.LogInformation("Folder {Path} exists, looking for files", path); + foreach (var f in Directory.GetFiles(path, "*.json", SearchOption.AllDirectories)) + { + var relativePath = Path.GetRelativePath($"{Directory.GetCurrentDirectory()}/{options.Value.CachePath}", f); + logger.LogInformation("Found file {RelativePath}", relativePath); + yield return await Task.FromResult(new BtmsBlobItem { Name = relativePath }); + } + } + else{ + logger.LogWarning("Cannot scan folder {Path} as it doesn't exist", path); + } } } - public Task GetResource(IBlobItem item, CancellationToken cancellationToken) + public async Task GetResource(IBlobItem item, CancellationToken cancellationToken) { + if (!options.Value.CacheReadEnabled) + { + var blob =blobService.GetResource(item, cancellationToken); + + if (options.Value.CacheWriteEnabled) + { + item.Content = blob.Result; + await CreateBlobAsync(item); + } + + return blob.Result; + } + var filePath = $"{options.Value.CachePath}/{item.Name}"; logger.LogInformation("GetResource {FilePath}", filePath); - return Task.Run(() => File.ReadAllText(filePath), cancellationToken); + return File.ReadAllText(filePath); } public async Task CreateBlobsAsync(IBlobItem[] items) diff --git a/Btms.Business.Tests/Commands/SyncDecisionsCommandTests.cs b/Btms.Business.Tests/Commands/SyncDecisionsCommandTests.cs index def5cb5..82d9d7a 100644 --- a/Btms.Business.Tests/Commands/SyncDecisionsCommandTests.cs +++ b/Btms.Business.Tests/Commands/SyncDecisionsCommandTests.cs @@ -45,7 +45,7 @@ public async Task WhenDecisionBlobsExist_ThenTheyShouldBePlacedOnInternalBus() await handler.Handle(command, CancellationToken.None); // ASSERT - await bus.Received(1).Publish(Arg.Any(), "DECISIONS", + await bus.Received(1).Publish(Arg.Any(), "DECISIONS", Arg.Any>(), Arg.Any()); } } diff --git a/Btms.Business.Tests/Services/Decisions/NoMatchDecisionsTest.cs b/Btms.Business.Tests/Services/Decisions/NoMatchDecisionsTest.cs index 611052f..2931143 100644 --- a/Btms.Business.Tests/Services/Decisions/NoMatchDecisionsTest.cs +++ b/Btms.Business.Tests/Services/Decisions/NoMatchDecisionsTest.cs @@ -35,7 +35,8 @@ public async Task WhenClearanceRequest_HasNotMatch_ThenDecisionCodeShouldBeNoMat decisionResult.Should().NotBeNull(); decisionResult.Decisions.Count.Should().Be(1); decisionResult.Decisions[0].DecisionCode.Should().Be(DecisionCode.X00); - await publishBus.Received().Publish(Arg.Any(), Arg.Any(), + + await publishBus.Received().Publish(Arg.Any(), Arg.Any(), Arg.Any>(), Arg.Any()); await Task.CompletedTask; } @@ -48,9 +49,9 @@ private static List GenerateMovements() var generatorResult = generator.Generate(1, 1, DateTime.UtcNow, config); - return generatorResult.ClearanceRequests.Select(x => + return generatorResult.Select(x => { - var internalClearanceRequest = AlvsClearanceRequestMapper.Map(x); + var internalClearanceRequest = AlvsClearanceRequestMapper.Map((AlvsClearanceRequest)x); return MovementPreProcessor.BuildMovement(internalClearanceRequest); }).ToList(); } diff --git a/Btms.Business.Tests/Services/Matching/MatchingServiceTests.cs b/Btms.Business.Tests/Services/Matching/MatchingServiceTests.cs index 9a6f19c..b55825c 100644 --- a/Btms.Business.Tests/Services/Matching/MatchingServiceTests.cs +++ b/Btms.Business.Tests/Services/Matching/MatchingServiceTests.cs @@ -2,6 +2,7 @@ using Btms.Business.Services.Matching; using Btms.Model; using Btms.Model.Ipaffs; +using Btms.Types.Alvs; using Btms.Types.Alvs.Mapping; using Btms.Types.Ipaffs.Mapping; using FluentAssertions; @@ -52,9 +53,9 @@ private static List GenerateMovements() var generatorResult = generator.Generate(1, 1, DateTime.UtcNow, config); - return generatorResult.ClearanceRequests.Select(x => + return generatorResult.Select(x => { - var internalClearanceRequest = AlvsClearanceRequestMapper.Map(x); + var internalClearanceRequest = AlvsClearanceRequestMapper.Map((AlvsClearanceRequest)x); return MovementPreProcessor.BuildMovement(internalClearanceRequest); }).ToList(); } @@ -67,15 +68,31 @@ private static (List Notifications, List Movements var generatorResult = generator.Generate(1, 1, DateTime.UtcNow, config); - var movements = generatorResult.ClearanceRequests.Select(x => - { - var internalClearanceRequest = AlvsClearanceRequestMapper.Map(x); - return MovementPreProcessor.BuildMovement(internalClearanceRequest); - }).ToList(); - - var notifications = generatorResult.ImportNotifications.Select(ImportNotificationMapper.Map).ToList(); - - return new ValueTuple, List>(notifications, movements); + var messages = generatorResult.Aggregate( + new { Notifications = new List(), Movements = new List(), }, (memo, x) => + { + switch (x) + { + case null: + throw new ArgumentNullException(); + + case Btms.Types.Ipaffs.ImportNotification n: + var internalNotification = ImportNotificationMapper.Map(n); + memo.Notifications.Add(internalNotification); + break; + case AlvsClearanceRequest cr: + + var internalClearanceRequest = AlvsClearanceRequestMapper.Map(cr); + memo.Movements.Add(MovementPreProcessor.BuildMovement(internalClearanceRequest)); + break; + default: + throw new ArgumentException($"Unexpected type {x.GetType().Name}"); + } + + return memo; + }); + + return new ValueTuple, List>(messages.Notifications, messages.Movements); } diff --git a/Btms.Business/Commands/SyncDecisionsCommand.cs b/Btms.Business/Commands/SyncDecisionsCommand.cs index c9a5497..ada3ff8 100644 --- a/Btms.Business/Commands/SyncDecisionsCommand.cs +++ b/Btms.Business/Commands/SyncDecisionsCommand.cs @@ -26,7 +26,7 @@ public override async Task Handle(SyncDecisionsCommand request, CancellationToke var rootFolder = string.IsNullOrEmpty(request.RootFolder) ? Options.DmpBlobRootFolder : request.RootFolder; - await SyncBlobPaths(request.SyncPeriod, "DECISIONS", request.JobId, + await SyncBlobPaths(request.SyncPeriod, "DECISIONS", request.JobId, cancellationToken,$"{rootFolder}/DECISIONS"); } } diff --git a/Btms.Business/Pipelines/PreProcessing/MovementPreProcessor.cs b/Btms.Business/Pipelines/PreProcessing/MovementPreProcessor.cs index 85c3609..9f39852 100644 --- a/Btms.Business/Pipelines/PreProcessing/MovementPreProcessor.cs +++ b/Btms.Business/Pipelines/PreProcessing/MovementPreProcessor.cs @@ -24,7 +24,8 @@ public async Task> Process(PreProcessingContext> Process(PreProcessingContext diff --git a/Btms.Business/Services/Decisions/DecisionMessageBuilder.cs b/Btms.Business/Services/Decisions/DecisionMessageBuilder.cs index 49f1751..e91d4e6 100644 --- a/Btms.Business/Services/Decisions/DecisionMessageBuilder.cs +++ b/Btms.Business/Services/Decisions/DecisionMessageBuilder.cs @@ -1,14 +1,15 @@ using Btms.Model; using Btms.Model.Ipaffs; using Btms.Types.Alvs; +using Decision = Btms.Types.Alvs.Decision; namespace Btms.Business.Services.Decisions; public static class DecisionMessageBuilder { - public static Task> Build(DecisionContext decisionContext, DecisionResult decisionResult) + public static Task> Build(DecisionContext decisionContext, DecisionResult decisionResult) { - var list = new List(); + var list = new List(); var movements = decisionResult.Decisions.GroupBy(x => x.MovementId).ToList(); @@ -16,7 +17,7 @@ public static Task> Build(DecisionContext decisionCon { var movement = decisionContext.Movements.First(x => x.Id == movementGroup.Key); var messageNumber = movement is { Decisions: null } ? 1 : movement.Decisions.Count + 1; - var decisionMessage = new AlvsClearanceRequest() + var decisionMessage = new Decision() { ServiceHeader = BuildServiceHeader(), Header = BuildHeader(movement, messageNumber), @@ -48,10 +49,7 @@ private static Header BuildHeader(Movement movement, int messageNumber) DecisionNumber = messageNumber }; } - - - - + private static IEnumerable BuildItems(Movement movement, IGrouping movementGroup) { var itemGroups = movementGroup.GroupBy(x => x.ItemNumber); diff --git a/Btms.Common/Extensions/DateTimeExtensions.cs b/Btms.Common/Extensions/DateTimeExtensions.cs index 1a453f3..9c9623a 100644 --- a/Btms.Common/Extensions/DateTimeExtensions.cs +++ b/Btms.Common/Extensions/DateTimeExtensions.cs @@ -69,9 +69,10 @@ private static int CreateRandomInt(int min, int max) return Random.Shared.Next(min, max); } - public static DateTime RandomTime(this DateTime dt) + public static DateTime RandomTime(this DateTime dt, int maxHour = 23) { - return new DateTime(dt.Year, dt.Month, dt.Day, CreateRandomInt(0,23), CreateRandomInt(0, 60), CreateRandomInt(0, 60), dt.Kind); + return new DateTime(dt.Year, dt.Month, dt.Day, + CreateRandomInt(0, maxHour), CreateRandomInt(0, 60), CreateRandomInt(0, 60), dt.Kind); } public static DateOnly ToDate(this DateTime val) diff --git a/Btms.Consumers.Tests/ClearanceRequestConsumerTests.cs b/Btms.Consumers.Tests/ClearanceRequestConsumerTests.cs index da8edc4..f7219ef 100644 --- a/Btms.Consumers.Tests/ClearanceRequestConsumerTests.cs +++ b/Btms.Consumers.Tests/ClearanceRequestConsumerTests.cs @@ -30,7 +30,7 @@ public async Task WhenPreProcessingSucceeds_AndLastAuditEntryIsLinked_ThenLinkSh var movement = MovementPreProcessor.BuildMovement(AlvsClearanceRequestMapper.Map(clearanceRequest)); - movement.Update(AuditEntry.CreateLinked("Test", 1, DateTime.Now)); + movement.Update(AuditEntry.CreateLinked("Test", 1)); var mockLinkingService = Substitute.For(); var decisionService = Substitute.For(); @@ -67,7 +67,7 @@ public async Task WhenPreProcessingSucceeds_AndLastAuditEntryIsCreated_ThenLinkS var movement = MovementPreProcessor.BuildMovement(AlvsClearanceRequestMapper.Map(clearanceRequest)); - movement.Update(AuditEntry.CreateCreatedEntry(movement,"Test", 1, DateTime.Now)); + movement.Update(AuditEntry.CreateCreatedEntry(movement,"Test", 1, DateTime.Now, AuditEntry.CreatedByCds)); var mockLinkingService = Substitute.For(); var decisionService = Substitute.For(); diff --git a/Btms.Consumers.Tests/NotificationsConsumerTests.cs b/Btms.Consumers.Tests/NotificationsConsumerTests.cs index 067e95d..9d6b11b 100644 --- a/Btms.Consumers.Tests/NotificationsConsumerTests.cs +++ b/Btms.Consumers.Tests/NotificationsConsumerTests.cs @@ -27,7 +27,7 @@ public async Task WhenPreProcessingSucceeds_AndLastAuditEntryIsLinked_ThenLinkSh // ARRANGE var notification = CreateImportNotification(); var modelNotification = notification.MapWithTransform(); - modelNotification.Changed(AuditEntry.CreateLinked("Test", 1, DateTime.Now)); + modelNotification.Changed(AuditEntry.CreateLinked("Test", 1)); var mockLinkingService = Substitute.For(); var decisionService = Substitute.For(); var matchingService = Substitute.For(); @@ -57,7 +57,7 @@ public async Task WhenPreProcessingSucceeds_AndLastAuditEntryIsCreated_ThenLinkS // ARRANGE var notification = CreateImportNotification(); var modelNotification = notification.MapWithTransform(); - modelNotification.Changed(AuditEntry.CreateCreatedEntry(modelNotification, "Test", 1, DateTime.Now)); + modelNotification.Changed(AuditEntry.CreateCreatedEntry(modelNotification, "Test", 1, DateTime.Now, AuditEntry.CreatedByIpaffs)); var mockLinkingService = Substitute.For(); var decisionService = Substitute.For(); var matchingService = Substitute.For(); diff --git a/Btms.Consumers/AlvsClearanceRequestConsumer.cs b/Btms.Consumers/AlvsClearanceRequestConsumer.cs index 404ea7b..f209a39 100644 --- a/Btms.Consumers/AlvsClearanceRequestConsumer.cs +++ b/Btms.Consumers/AlvsClearanceRequestConsumer.cs @@ -42,10 +42,10 @@ public async Task OnHandle(AlvsClearanceRequest message) Context.Linked(); } - var matchResult = await matchingService.Process( - new MatchingContext(linkResult.Notifications, linkResult.Movements), Context.CancellationToken); + var matchResult = await matchingService.Process( + new MatchingContext(linkResult.Notifications, linkResult.Movements), Context.CancellationToken); - await decisionService.Process(new DecisionContext(linkResult.Notifications, linkResult.Movements, matchResult, true), Context.CancellationToken); + await decisionService.Process(new DecisionContext(linkResult.Notifications, linkResult.Movements, matchResult, true), Context.CancellationToken); } } } diff --git a/Btms.Consumers/ConsumerOptions.cs b/Btms.Consumers/ConsumerOptions.cs index 7b64d5a..f9fdde9 100644 --- a/Btms.Consumers/ConsumerOptions.cs +++ b/Btms.Consumers/ConsumerOptions.cs @@ -4,6 +4,7 @@ public class ConsumerOptions { public const string SectionName = nameof(ConsumerOptions); + public bool EnableBlockingPublish { get; set; } = false; public int InMemoryNotifications { get; set; } = 2; public int InMemoryGmrs { get; set; } = 2; public int InMemoryClearanceRequests { get; set; } = 2; diff --git a/Btms.Consumers/DecisionsConsumer.cs b/Btms.Consumers/DecisionsConsumer.cs index b1fe7e0..8ce457c 100644 --- a/Btms.Consumers/DecisionsConsumer.cs +++ b/Btms.Consumers/DecisionsConsumer.cs @@ -6,11 +6,11 @@ namespace Btms.Consumers; public class DecisionsConsumer(IMongoDbContext dbContext) - : IConsumer, IConsumerWithContext + : IConsumer, IConsumerWithContext { - public async Task OnHandle(AlvsClearanceRequest message) + public async Task OnHandle(Decision message) { - var internalClearanceRequest = AlvsClearanceRequestMapper.Map(message); + var internalClearanceRequest = DecisionMapper.Map(message); var existingMovement = await dbContext.Movements.Find(message.Header!.EntryReference!); if (existingMovement != null) diff --git a/Btms.Consumers/Extensions/ServiceCollectionExtensions.cs b/Btms.Consumers/Extensions/ServiceCollectionExtensions.cs index 799d2c3..b643580 100644 --- a/Btms.Consumers/Extensions/ServiceCollectionExtensions.cs +++ b/Btms.Consumers/Extensions/ServiceCollectionExtensions.cs @@ -10,6 +10,7 @@ using SlimMessageBus.Host.Interceptor; using SlimMessageBus.Host.Memory; using AlvsClearanceRequest = Btms.Types.Alvs.AlvsClearanceRequest; +using Decision = Btms.Types.Alvs.Decision; namespace Btms.Consumers.Extensions { @@ -41,7 +42,7 @@ public static IServiceCollection AddConsumers(this IServiceCollection services, { cbb.WithProviderMemory(cfg => { - cfg.EnableBlockingPublish = false; + cfg.EnableBlockingPublish = consumerOpts.EnableBlockingPublish; cfg.EnableMessageHeaders = true; }) .AddServicesFromAssemblyContaining( @@ -64,7 +65,8 @@ public static IServiceCollection AddConsumers(this IServiceCollection services, x.Instances(consumerOpts.InMemoryClearanceRequests); x.Topic("CLEARANCEREQUESTS").WithConsumer(); }) - .Consume(x => + .Produce(x => x.DefaultTopic(nameof(Decision))) + .Consume(x => { x.Instances(consumerOpts.InMemoryDecisions); x.Topic("DECISIONS").WithConsumer(); diff --git a/Btms.Consumers/GmrConsumer.cs b/Btms.Consumers/GmrConsumer.cs index 98832dc..be078f4 100644 --- a/Btms.Consumers/GmrConsumer.cs +++ b/Btms.Consumers/GmrConsumer.cs @@ -19,7 +19,7 @@ public async Task OnHandle(SearchGmrsForDeclarationIdsResponse message) if (existingGmr is null) { var auditEntry = - AuditEntry.CreateCreatedEntry(internalGmr, auditId!, 1, gmr.UpdatedSource); + AuditEntry.CreateCreatedEntry(internalGmr, auditId!, 1, gmr.UpdatedSource, AuditEntry.CreatedByGvms); internalGmr.AuditEntries.Add(auditEntry); await dbContext.Gmrs.Insert(internalGmr); } @@ -33,7 +33,8 @@ public async Task OnHandle(SearchGmrsForDeclarationIdsResponse message) current: internalGmr, id: auditId!, version: internalGmr.AuditEntries.Count + 1, - lastUpdated: gmr.UpdatedSource); + lastUpdated: gmr.UpdatedSource, + AuditEntry.CreatedByGvms); internalGmr.AuditEntries.Add(auditEntry); await dbContext.Gmrs.Update(internalGmr, existingGmr._Etag); } diff --git a/Btms.Consumers/NotificationConsumer.cs b/Btms.Consumers/NotificationConsumer.cs index 13fc6c5..8a17ab8 100644 --- a/Btms.Consumers/NotificationConsumer.cs +++ b/Btms.Consumers/NotificationConsumer.cs @@ -42,10 +42,10 @@ public async Task OnHandle(ImportNotification message) Context.Linked(); } - var matchResult = await matchingService.Process( - new MatchingContext(linkResult.Notifications, linkResult.Movements), Context.CancellationToken); + var matchResult = await matchingService.Process( + new MatchingContext(linkResult.Notifications, linkResult.Movements), Context.CancellationToken); - await decisionService.Process(new DecisionContext(linkResult.Notifications, linkResult.Movements, matchResult), Context.CancellationToken); + await decisionService.Process(new DecisionContext(linkResult.Notifications, linkResult.Movements, matchResult), Context.CancellationToken); } } diff --git a/Btms.Model/Auditing/AuditEntry.cs b/Btms.Model/Auditing/AuditEntry.cs index 2c56320..7eb9f92 100644 --- a/Btms.Model/Auditing/AuditEntry.cs +++ b/Btms.Model/Auditing/AuditEntry.cs @@ -8,9 +8,13 @@ namespace Btms.Model.Auditing; public class AuditEntry { - private const string CreatedBySystem = "System"; + public const string CreatedBySystem = "Btms"; + public const string CreatedByIpaffs = "Ipaffs"; + public const string CreatedByAlvs = "Alvs"; + public const string CreatedByCds = "Cds"; + public const string CreatedByGvms = "Gvms"; public string Id { get; set; } = default!; - public int Version { get; set; } + public int? Version { get; set; } public string CreatedBy { get; set; } = default!; @@ -21,6 +25,8 @@ public class AuditEntry public string Status { get; set; } = default!; public List Diff { get; set; } = new(); + + public Dictionary> Context { get; set; } = new(); public bool IsCreatedOrUpdated() { @@ -39,27 +45,27 @@ public bool IsUpdated() public static AuditEntry Create(T previous, T current, string id, int version, DateTime? lastUpdated, - string lastUpdatedBy, string status) + string lastUpdatedBy, string status, string source) { var node1 = JsonNode.Parse(previous.ToJsonString()); var node2 = JsonNode.Parse(current.ToJsonString()); - return CreateInternal(node1!, node2!, id, version, lastUpdated, status); + return CreateInternal(node1!, node2!, id, version, lastUpdated, status, source); } - public static AuditEntry CreateUpdated(T previous, T current, string id, int version, DateTime? lastUpdated) + public static AuditEntry CreateUpdated(T previous, T current, string id, int version, DateTime? lastUpdated, string source) { - return Create(previous, current, id, version, lastUpdated, CreatedBySystem, "Updated"); + return Create(previous, current, id, version, lastUpdated, CreatedBySystem, "Updated", source); } - public static AuditEntry CreateUpdated(ChangeSet changeSet, string id, int version, DateTime? lastUpdated) + public static AuditEntry CreateUpdated(ChangeSet changeSet, string id, int version, DateTime? lastUpdated, string source) { var auditEntry = new AuditEntry { Id = id, Version = version, CreatedSource = lastUpdated, - CreatedBy = CreatedBySystem, + CreatedBy = source, CreatedLocal = DateTime.UtcNow, Status = "Updated" }; @@ -72,74 +78,75 @@ public static AuditEntry CreateUpdated(ChangeSet changeSet, string id, int versi return auditEntry; } - public static AuditEntry CreateCreatedEntry(T current, string id, int version, DateTime? lastUpdated) + public static AuditEntry CreateCreatedEntry(T current, string id, int version, DateTime? lastUpdated, string source) { return new AuditEntry { Id = id, Version = version, CreatedSource = lastUpdated, - CreatedBy = CreatedBySystem, + CreatedBy = source, CreatedLocal = DateTime.UtcNow, Status = "Created" }; } - public static AuditEntry CreateSkippedVersion(string id, int version, DateTime? lastUpdated) + public static AuditEntry CreateSkippedVersion(string id, int version, DateTime? lastUpdated, string source) { return new AuditEntry { Id = id, Version = version, CreatedSource = lastUpdated, - CreatedBy = CreatedBySystem, + CreatedBy = source, CreatedLocal = DateTime.UtcNow, Status = "Updated" }; } - public static AuditEntry CreateLinked(string id, int version, DateTime? lastUpdated) + public static AuditEntry CreateLinked(string id, int version) { + var t = DateTime.UtcNow; return new AuditEntry { Id = id, - Version = version, - CreatedSource = lastUpdated, + CreatedSource = t, CreatedBy = CreatedBySystem, - CreatedLocal = DateTime.UtcNow, + CreatedLocal = t, Status = "Linked" }; } - public static AuditEntry CreateMatch(string id, int version, DateTime? lastUpdated) + public static AuditEntry CreateMatch(string id, int version) { + var t = DateTime.UtcNow; return new AuditEntry { Id = id, - Version = version, - CreatedSource = lastUpdated, + CreatedSource = t, CreatedBy = CreatedBySystem, - CreatedLocal = DateTime.UtcNow, + CreatedLocal = t, Status = "Matched" }; } public static AuditEntry CreateDecision(string id, int version, - DateTime? lastUpdated, string lastUpdatedBy) + DateTime? lastUpdated, string lastUpdatedBy, Dictionary> context, bool isAlvs) { return new AuditEntry() { Id = id, - Version = version, CreatedSource = lastUpdated, - CreatedBy = CreatedBySystem, + CreatedBy = isAlvs ? CreatedByAlvs : CreatedBySystem, CreatedLocal = DateTime.UtcNow, - Status = "Decision" + Status = "Decision", + Context = context + }; } private static AuditEntry CreateInternal(JsonNode previous, JsonNode current, string id, int version, - DateTime? lastUpdated, string status) + DateTime? lastUpdated, string status, string source) { var diff = previous.CreatePatch(current); @@ -148,7 +155,7 @@ private static AuditEntry CreateInternal(JsonNode previous, JsonNode current, st Id = id, Version = version, CreatedSource = lastUpdated, - CreatedBy = CreatedBySystem, + CreatedBy = source, CreatedLocal = DateTime.UtcNow, Status = status }; diff --git a/Btms.Model/Ipaffs/ImportNotification.cs b/Btms.Model/Ipaffs/ImportNotification.cs index 3c94ae1..c9ac2a0 100644 --- a/Btms.Model/Ipaffs/ImportNotification.cs +++ b/Btms.Model/Ipaffs/ImportNotification.cs @@ -144,7 +144,7 @@ public void AddRelationship(TdmRelationshipObject relationship) if (linked) { - AuditEntries.Add(AuditEntry.CreateLinked(string.Empty, Version.GetValueOrDefault(), UpdatedSource)); + AuditEntries.Add(AuditEntry.CreateLinked(string.Empty, Version.GetValueOrDefault())); } } @@ -159,7 +159,8 @@ public void Create(string auditId) this, auditId, Version.GetValueOrDefault(), - UpdatedSource); + UpdatedSource, + AuditEntry.CreatedByIpaffs); Changed(auditEntry); } @@ -168,7 +169,8 @@ public void Skipped(string auditId, int version) var auditEntry = AuditEntry.CreateSkippedVersion( auditId, version, - UpdatedSource); + UpdatedSource, + AuditEntry.CreatedByIpaffs); Changed(auditEntry); } @@ -177,7 +179,8 @@ public void Update(string auditId, ChangeSet changeSet) var auditEntry = AuditEntry.CreateUpdated(changeSet, auditId, Version.GetValueOrDefault(), - UpdatedSource); + UpdatedSource, + AuditEntry.CreatedByIpaffs); Changed(auditEntry); } diff --git a/Btms.Model/Movement.cs b/Btms.Model/Movement.cs index 775827a..0b70088 100644 --- a/Btms.Model/Movement.cs +++ b/Btms.Model/Movement.cs @@ -111,7 +111,7 @@ public void AddRelationship(TdmRelationshipObject relationship) if (linked) { - AuditEntries.Add(AuditEntry.CreateLinked(String.Empty, this.AuditEntries.FirstOrDefault()?.Version ?? 1, UpdatedSource)); + AuditEntries.Add(AuditEntry.CreateLinked(String.Empty, this.AuditEntries.FirstOrDefault()?.Version ?? 1)); } } @@ -136,11 +136,23 @@ public bool MergeDecision(string path, AlvsClearanceRequest clearanceRequest) } } + var decisionAuditContext = new Dictionary>(); + decisionAuditContext.Add("movements", new Dictionary() + { + { clearanceRequest.Header!.EntryReference!, clearanceRequest.Header!.EntryVersionNumber!.ToString()! } + }); + decisionAuditContext.Add("importNotifications", new Dictionary() + { + { "todo", "todo" } + }); + var auditEntry = AuditEntry.CreateDecision( BuildNormalizedDecisionPath(path), clearanceRequest.Header!.EntryVersionNumber.GetValueOrDefault(), clearanceRequest.ServiceHeader!.ServiceCalled, - clearanceRequest.Header.DeclarantName!); + clearanceRequest.Header.DeclarantName!, + decisionAuditContext, + clearanceRequest.ServiceHeader?.SourceSystem != "BTMS"); Decisions ??= []; Decisions.Add(clearanceRequest); diff --git a/Btms.SyncJob/Extensions/ServiceProviderExtensions.cs b/Btms.SyncJob/Extensions/ServiceProviderExtensions.cs new file mode 100644 index 0000000..2829093 --- /dev/null +++ b/Btms.SyncJob/Extensions/ServiceProviderExtensions.cs @@ -0,0 +1,24 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; + +namespace Btms.SyncJob.Extensions; + +public static class ServiceProviderExtensions +{ + public static async Task WaitOnAllJobs(this IServiceProvider provider, ILogger logger) + { + var store = provider.GetService()!; + + var complete = false; + while (!complete) + { + var jobs = store.GetJobs(); + // logger.LogInformation(jobs.ToJsonString()); + logger.LogInformation("{jobs} found.", jobs.Count); + + complete = true; + } + + return await Task.FromResult(true); + } +} \ No newline at end of file diff --git a/Btms.Tests.Common/Btms.Tests.Common.csproj b/Btms.Tests.Common/Btms.Tests.Common.csproj new file mode 100644 index 0000000..756957c --- /dev/null +++ b/Btms.Tests.Common/Btms.Tests.Common.csproj @@ -0,0 +1,18 @@ + + + + net8.0 + enable + enable + + + + + + + + + + + + diff --git a/Btms.Tests.Common/Class1.cs b/Btms.Tests.Common/Class1.cs new file mode 100644 index 0000000..70636b9 --- /dev/null +++ b/Btms.Tests.Common/Class1.cs @@ -0,0 +1,5 @@ +namespace Btms.Tests.Common; + +public class Class1 +{ +} diff --git a/Btms.Types.Alvs.Mapping.V1/DecisionMapper.g.cs b/Btms.Types.Alvs.Mapping.V1/DecisionMapper.g.cs new file mode 100644 index 0000000..4216d01 --- /dev/null +++ b/Btms.Types.Alvs.Mapping.V1/DecisionMapper.g.cs @@ -0,0 +1,32 @@ +//------------------------------------------------------------------------------ +// + // This code was generated from a template. + // + // Manual changes to this file may cause unexpected behavior in your application. + // Manual changes to this file will be overwritten if the code is regenerated. + // +// +//------------------------------------------------------------------------------ +#nullable enable + + +namespace Btms.Types.Alvs.Mapping; + +public static class DecisionMapper +{ + public static Btms.Model.Alvs.AlvsClearanceRequest Map(Btms.Types.Alvs.Decision from) + { + if (from is null) + { + return default!; + } + + var to = new Btms.Model.Alvs.AlvsClearanceRequest(); + to.ServiceHeader = ServiceHeaderMapper.Map(from?.ServiceHeader!); + to.Header = HeaderMapper.Map(from?.Header!); + to.Items = from?.Items?.Select(x => ItemsMapper.Map(x)).ToArray(); + return to; + } +} + + diff --git a/Btms.Types.Alvs.V1/ClearanceRequestExtensions.cs b/Btms.Types.Alvs.V1/ClearanceRequestExtensions.cs index 0c85cc8..e31dd49 100644 --- a/Btms.Types.Alvs.V1/ClearanceRequestExtensions.cs +++ b/Btms.Types.Alvs.V1/ClearanceRequestExtensions.cs @@ -44,6 +44,6 @@ public override DateTime Read(ref Utf8JsonReader reader, Type typeToConvert, Jso public override void Write(Utf8JsonWriter writer, DateTime value, JsonSerializerOptions options) { - writer.WriteStringValue(new DateTimeOffset(value).ToUnixTimeMilliseconds().ToString()); + writer.WriteNumberValue(new DateTimeOffset(value).ToUnixTimeMilliseconds()); } } \ No newline at end of file diff --git a/Btms.Types.Alvs.V1/Decision.cs b/Btms.Types.Alvs.V1/Decision.cs new file mode 100644 index 0000000..3ab8861 --- /dev/null +++ b/Btms.Types.Alvs.V1/Decision.cs @@ -0,0 +1,42 @@ + +#nullable enable + +using System.Text.Json.Serialization; +using System.Dynamic; + + +namespace Btms.Types.Alvs; + +/// +/// This is a copy of the AlvsClearanceRequest +/// As a temporary measure to allow us to distinguish between the two types when +/// selecting consumers via the DI container. We'll also start to +/// Plan is to follow the same approach as other types (currently generated) in time. +/// +public class Decision // +{ + + + /// + /// + /// + [JsonPropertyName("serviceHeader")] + public ServiceHeader? ServiceHeader { get; set; } + + + /// + /// + /// + [JsonPropertyName("header")] + public Header? Header { get; set; } + + + /// + /// + /// + [JsonPropertyName("items")] + public Items[]? Items { get; set; } + + } + + diff --git a/TestDataGenerator.Tests/ClearanceRequestBuilderTests.cs b/TestDataGenerator.Tests/ClearanceRequestBuilderTests.cs index 6e81162..24ff92a 100644 --- a/TestDataGenerator.Tests/ClearanceRequestBuilderTests.cs +++ b/TestDataGenerator.Tests/ClearanceRequestBuilderTests.cs @@ -20,7 +20,7 @@ public void WithEntryDate_ShouldSet() { var date = DateTime.Today.AddDays(-5); var builder = ClearanceRequestBuilder.Default(); - builder.WithEntryDate(date); + builder.WithCreationDate(date); var cr = builder.Build(); cr.ServiceHeader!.ServiceCallTimestamp.ToDate().Should().Be(date.ToDate()); diff --git a/TestDataGenerator.Tests/ImportNotificationBuilderTests.cs b/TestDataGenerator.Tests/ImportNotificationBuilderTests.cs index d39de6e..b98e182 100644 --- a/TestDataGenerator.Tests/ImportNotificationBuilderTests.cs +++ b/TestDataGenerator.Tests/ImportNotificationBuilderTests.cs @@ -39,7 +39,7 @@ public void WithEntryDate_ShouldSet() { var date = DateTime.Today.AddDays(-5); var builder = ImportNotificationBuilder.Default(); - builder.WithEntryDate(date); + builder.WithCreationDate(date); var notification = builder.Build(); notification.LastUpdated.ToDate().Should().Be(date.ToDate()); diff --git a/TestDataGenerator/ClearanceRequestBuilder.cs b/TestDataGenerator/ClearanceRequestBuilder.cs index 0f0d441..34294a0 100644 --- a/TestDataGenerator/ClearanceRequestBuilder.cs +++ b/TestDataGenerator/ClearanceRequestBuilder.cs @@ -44,11 +44,19 @@ public ClearanceRequestBuilder WithReferenceNumber(string chedReference) }); } - public ClearanceRequestBuilder WithEntryDate(DateTime entryDate) + public ClearanceRequestBuilder WithCreationDate(DateTime entryDate, bool randomTime = true) { - return Do(x => x.ServiceHeader!.ServiceCallTimestamp = entryDate.RandomTime()); + var entry = randomTime ? + // We don't want documents created in the future! + entryDate.RandomTime(entryDate.Date == DateTime.Today ? DateTime.Now.AddHours(-2).Hour : 23) + : entryDate; + + return Do(x => x.ServiceHeader!.ServiceCallTimestamp = entry); + } + public ClearanceRequestBuilder WithEntryVersionNumber(int version) + { + return Do(x => x.Header!.EntryVersionNumber = version); } - public ClearanceRequestBuilder WithArrivalDateTimeOffset(DateOnly? date, TimeOnly? time, int maxHoursOffset = 12, int maxMinsOffset = 30) { @@ -65,7 +73,7 @@ public ClearanceRequestBuilder WithArrivalDateTimeOffset(DateOnly? date, Time } public ClearanceRequestBuilder WithItem(string documentCode, string commodityCode, string description, - int netWeight) + int netWeight, string checkCode = "H2019") { return Do(cr => { @@ -73,6 +81,7 @@ public ClearanceRequestBuilder WithItem(string documentCode, string commodity cr.Items![0].GoodsDescription = description; cr.Items![0].ItemNetMass = netWeight; cr.Items![0].Documents![0].DocumentCode = documentCode; + cr.Items![0].Checks![0].CheckCode = checkCode; }); } diff --git a/TestDataGenerator/DecisionBuilder.cs b/TestDataGenerator/DecisionBuilder.cs new file mode 100644 index 0000000..16a7eaa --- /dev/null +++ b/TestDataGenerator/DecisionBuilder.cs @@ -0,0 +1,87 @@ +using Btms.Common.Extensions; +using Btms.Model; +using Btms.Types.Alvs; +using TestDataGenerator.Helpers; + +namespace TestDataGenerator; + +public class DecisionBuilder(string file) : DecisionBuilder(file); + +public class DecisionBuilder : BuilderBase> + where T : Decision, new() +{ + private DecisionBuilder() + { + } + + protected DecisionBuilder(string file) : base(file) + { + } + + public static DecisionBuilder Default() + { + return new DecisionBuilder(); + } + + public static DecisionBuilder FromFile(string file) + { + return new DecisionBuilder(file); + } + + public DecisionBuilder WithReferenceNumber(string chedReference) + { + var id = MatchIdentifier.FromNotification(chedReference); + return Do(x => + { + x.Header!.EntryReference = id.AsCdsEntryReference(); + x.Header!.DeclarationUcr = id.AsCdsDeclarationUcr(); + x.Header!.MasterUcr = id.AsCdsMasterUcr(); + }); + } + + public DecisionBuilder WithDecisionNumber(int number) + { + return Do(x => x.Header!.DecisionNumber = number); + } + + public DecisionBuilder WithCreationDate(DateTime entryDate, bool randomTime = true) + { + var entry = randomTime ? + // We don't want documents created in the future! + entryDate.RandomTime(entryDate.Date == DateTime.Today ? DateTime.Now.AddHours(-2).Hour : 23) + : entryDate; + + return Do(x => x.ServiceHeader!.ServiceCallTimestamp = entry); + } + public DecisionBuilder WithEntryVersionNumber(int version) + { + return Do(x => x.Header!.EntryVersionNumber = version); + } + + public DecisionBuilder WithItemAndCheck(int item, string checkCode, string decisionCode) + { + return Do(dec => + { + dec.Items![0].ItemNumber = item; + dec.Items![0].Checks![0].CheckCode = checkCode; + dec.Items![0].Checks![0].DecisionCode = decisionCode; + }); + } + + protected override DecisionBuilder Validate() + { + return Do(cr => + { + cr.ServiceHeader!.ServiceCallTimestamp.AssertHasValue("Decision ServiceCallTimestamp missing"); + cr.Header!.EntryReference.AssertHasValue("Decision EntryReference missing"); + cr.Header!.DecisionNumber.AssertHasValue("Decision DecisionNumber missing"); + + Array.ForEach(cr.Items!, i => + Array.ForEach(i.Checks!, c => + { + c.CheckCode.AssertHasValue(); + c.DecisionCode.AssertHasValue(); + })); + }); + } +} \ No newline at end of file diff --git a/TestDataGenerator/Generator.cs b/TestDataGenerator/Generator.cs index 319c35c..91df6fe 100644 --- a/TestDataGenerator/Generator.cs +++ b/TestDataGenerator/Generator.cs @@ -41,17 +41,18 @@ public async Task Generate(int scenario, ScenarioConfig config, string rootPath) private async Task InsertToBlobStorage(ScenarioGenerator.GeneratorResult result, string rootPath) { logger.LogInformation( - "Uploading {ImportNotificationsLength} Notification(s) and {ClearanceRequestsLength} Clearance Request(s) to blob storage", - result.ImportNotifications.Length, result.ClearanceRequests.Length); - - var importNotificationBlobItems = result.ImportNotifications.Select(n => - new BtmsBlobItem { Name = n.BlobPath(rootPath), Content = JsonSerializer.Serialize(n) }); - - var alvsClearanceRequestBlobItems = result.ClearanceRequests.Select(cr => - new BtmsBlobItem { Name = cr.BlobPath(rootPath), Content = JsonSerializer.Serialize(cr) }); + "Uploading {Count} messages to blob storage", + result.Count); + + var blobs = result.Select(r => new BtmsBlobItem + { + Name = r.BlobPath(rootPath), Content = JsonSerializer.Serialize(r) + }); - var success = await blobService.CreateBlobsAsync(importNotificationBlobItems - .Concat(alvsClearanceRequestBlobItems).ToArray()); + var success = await blobService.CreateBlobsAsync(blobs.ToArray()); + + // var success = await blobService.CreateBlobsAsync(importNotificationBlobItems + // .Concat(alvsClearanceRequestBlobItems).ToArray()); return success; } diff --git a/TestDataGenerator/Helpers/DataHelpers.cs b/TestDataGenerator/Helpers/DataHelpers.cs index 732f40f..92f1963 100644 --- a/TestDataGenerator/Helpers/DataHelpers.cs +++ b/TestDataGenerator/Helpers/DataHelpers.cs @@ -1,3 +1,4 @@ +using Btms.Common.Extensions; using Btms.Model; using Btms.Types.Ipaffs.V1.Extensions; using Btms.Types.Alvs; @@ -7,6 +8,21 @@ namespace TestDataGenerator.Helpers; public static class DataHelpers { + internal static string BlobPath(this object resource, string rootPath) + { + switch (resource) + { + case null: + throw new ArgumentNullException(); + case ImportNotification n: + return n.BlobPath(rootPath); + case AlvsClearanceRequest cr: + return cr.BlobPath(rootPath); + default: + throw new InvalidDataException($"Unexpected type {resource.GetType().Name}"); + } + } + internal static string BlobPath(this ImportNotification notification, string rootPath) { var dateString = notification.LastUpdated!.Value.ToString("yyyy/MM/dd"); @@ -22,9 +38,10 @@ internal static string DateRef(this DateTime created) internal static string BlobPath(this AlvsClearanceRequest clearanceRequest, string rootPath) { var dateString = clearanceRequest.ServiceHeader!.ServiceCallTimestamp!.Value.ToString("yyyy/MM/dd"); - + var subPath = clearanceRequest.Header!.DecisionNumber.HasValue() ? "DECISIONS" : "ALVS"; + return - $"{rootPath}/ALVS/{dateString}/{clearanceRequest.Header!.EntryReference!.Replace(".", "")}-{Guid.NewGuid()}.json"; + $"{rootPath}/{subPath}/{dateString}/{clearanceRequest.Header!.EntryReference!.Replace(".", "")}-{Guid.NewGuid()}.json"; } internal static string AsCdsEntryReference(this MatchIdentifier identifier) diff --git a/TestDataGenerator/ImportNotificationBuilder.cs b/TestDataGenerator/ImportNotificationBuilder.cs index a3a8514..e5e0936 100644 --- a/TestDataGenerator/ImportNotificationBuilder.cs +++ b/TestDataGenerator/ImportNotificationBuilder.cs @@ -68,9 +68,14 @@ public ImportNotificationBuilder WithReferenceNumber(ImportNotificationTypeEn x.ReferenceNumber = DataHelpers.GenerateReferenceNumber(chedType, scenario, created, item)); } - public ImportNotificationBuilder WithEntryDate(DateTime entryDate) + public ImportNotificationBuilder WithCreationDate(DateTime entryDate, bool randomTime = true) { - return Do(x => x.LastUpdated = entryDate.RandomTime()); + var entry = randomTime ? + // We don't want documents created in the future! + entryDate.RandomTime(entryDate.Date == DateTime.Today ? DateTime.Now.AddHours(-2).Hour : 23) + : entryDate; + + return Do(x => x.LastUpdated = entry); } public ImportNotificationBuilder WithRandomArrivalDateTime(int maxDays, int maxHours=6) diff --git a/TestDataGenerator/Program.cs b/TestDataGenerator/Program.cs index c35f996..aa0e27c 100644 --- a/TestDataGenerator/Program.cs +++ b/TestDataGenerator/Program.cs @@ -66,7 +66,11 @@ private static async Task Main(string[] args) { Dataset = "LoadTest-One", RootPath = "GENERATED-LOADTEST-ONE", - Scenarios = new[] { app.CreateScenarioConfig(1, 3) } + Scenarios = new[] + { + app.CreateScenarioConfig(1, 1), + app.CreateScenarioConfig(1, 1) + } }, new { diff --git a/TestDataGenerator/Properties/launchSettings.json b/TestDataGenerator/Properties/launchSettings.json index 31c3391..b44e8bf 100644 --- a/TestDataGenerator/Properties/launchSettings.json +++ b/TestDataGenerator/Properties/launchSettings.json @@ -12,6 +12,17 @@ "AZURE_TENANT_ID": "c9d74090-b4e6-4b04-981d-e6757a160812" } }, + "Generate LoadTest-One": { + "commandName": "Project", + "commandLineArgs": "LoadTest-One", + "environmentVariables": { + "DMP_ENVIRONMENT": "dev", + "DMP_SERVICE_BUS_NAME": "DEVTREINFSB1001", + "DMP_BLOB_STORAGE_NAME": "devdmpinfdl1001", + "DMP_SLOT": "1003", + "AZURE_TENANT_ID": "c9d74090-b4e6-4b04-981d-e6757a160812" + } + }, "Generate LoadTest-Basic": { "commandName": "Project", "commandLineArgs": "LoadTest-Basic", diff --git a/TestDataGenerator/README.md b/TestDataGenerator/README.md index 66411f6..257fe09 100644 --- a/TestDataGenerator/README.md +++ b/TestDataGenerator/README.md @@ -24,4 +24,12 @@ az storage blob upload-batch -d dmp-data-1001 --account-name snddmpinfdl1001 -s TestDataGenerator/.test-data-generator/GENERATED-LOADTEST-90Dx10k --destination-path GENERATED-LOADTEST-90Dx10k az storage blob upload-batch -d dmp-data-1001 --account-name snddmpinfdl1001 -s TestDataGenerator/.test-data-generator/GENERATED-LOADTEST-BASIC --destination-path GENERATED-LOADTEST-BASIC -az storage blob upload-batch -d dmp-data-1001 --account-name snddmpinfdl1001 -s TestDataGenerator/.test-data-generator/PRODREDACTED-20241204 --destination-path PRODREDACTED-20241204 \ No newline at end of file +az storage blob upload-batch -d dmp-data-1001 --account-name snddmpinfdl1001 -s TestDataGenerator/.test-data-generator/PRODREDACTED-20241204 --destination-path PRODREDACTED-20241204 + + +az storage blob directory list -c dmp-data-1001 -d --account-name snddmpinfdl1001 + +az storage fs directory list -f dmp-data-1001 -d --account-name snddmpinfdl1001 + + +az storage fs directory download -f dmp-data-1001 -s "PRODREDACTED-20241204" -d "TestDataGenerator/.test-data-generator" --recursive --account-name snddmpinfdl1001 diff --git a/TestDataGenerator/ScenarioGenerator.cs b/TestDataGenerator/ScenarioGenerator.cs index be15465..fe4aec7 100644 --- a/TestDataGenerator/ScenarioGenerator.cs +++ b/TestDataGenerator/ScenarioGenerator.cs @@ -1,6 +1,11 @@ +using System.Collections; +using AutoFixture; +using Btms.Common.Extensions; +using Btms.Model; using Btms.Types.Alvs; using Btms.Types.Ipaffs; using TestDataGenerator.Scenarios; +using Decision = Btms.Types.Alvs.Decision; namespace TestDataGenerator; @@ -27,9 +32,62 @@ internal ClearanceRequestBuilder GetClearanceRequestBuilder(string file) return builder; } - public class GeneratorResult + internal DecisionBuilder GetDecisionBuilder(string file) { - public ImportNotification[] ImportNotifications { get; set; } = default!; - public AlvsClearanceRequest[] ClearanceRequests { get; set; } = default!; + var fullPath = $"{_fullFolder}/{file}.json"; + var builder = new DecisionBuilder(fullPath); + + return builder; + } + + /// + /// A class to hold a list of message types we support. Would be nice to use something + /// other than object :| + /// + public class GeneratorResult : IEnumerable + { + public GeneratorResult(object[] initial) + { + foreach (var o in initial) + { + if (o is ImportNotification || o is AlvsClearanceRequest || o is Decision) + { + Messages.Add(o); + } + else + { + throw new Exception($"Unexpected GeneratorResult type {o.GetType().Name}"); + } + + } + } + + private List Messages { get; set; } = new List(); + + public void Add(ImportNotification[] importNotifications) + { + Messages.AddRange(importNotifications); + } + + public void Add(AlvsClearanceRequest[] clearanceRequests) + { + Messages.AddRange(clearanceRequests); + } + + public void Add(Btms.Types.Alvs.Decision[] decisions) + { + Messages.AddRange(decisions); + } + + IEnumerator IEnumerable.GetEnumerator() + { + return Messages.GetEnumerator(); + } + + public int Count => Messages.Count; + public IEnumerator GetEnumerator() + { + return Messages.GetEnumerator(); + } } } \ No newline at end of file diff --git a/TestDataGenerator/Scenarios/CRNoMatchScenarioGenerator.cs b/TestDataGenerator/Scenarios/CRNoMatchScenarioGenerator.cs index 5d15240..b7f7697 100644 --- a/TestDataGenerator/Scenarios/CRNoMatchScenarioGenerator.cs +++ b/TestDataGenerator/Scenarios/CRNoMatchScenarioGenerator.cs @@ -10,7 +10,7 @@ public class CrNoMatchScenarioGenerator(ILogger logg public override GeneratorResult Generate(int scenario, int item, DateTime entryDate, ScenarioConfig config) { var clearanceRequest = GetClearanceRequestBuilder("cr-one-item") - .WithEntryDate(entryDate) + .WithCreationDate(entryDate) .WithArrivalDateTimeOffset(DateTime.Today.ToDate(), DateTime.Now.ToTime()) .WithReferenceNumber(DataHelpers.GenerateReferenceNumber(ImportNotificationTypeEnum.Cveda, scenario, entryDate, item)) .WithRandomItems(10, 100) @@ -18,6 +18,6 @@ public override GeneratorResult Generate(int scenario, int item, DateTime entryD logger.LogInformation("Created {EntryReference}", clearanceRequest.Header!.EntryReference); - return new GeneratorResult { ClearanceRequests = [clearanceRequest], ImportNotifications = [] }; + return new GeneratorResult([clearanceRequest]); } } \ No newline at end of file diff --git a/TestDataGenerator/Scenarios/ChedAManyCommoditiesScenarioGenerator.cs b/TestDataGenerator/Scenarios/ChedAManyCommoditiesScenarioGenerator.cs index 843c50a..77ebc4b 100644 --- a/TestDataGenerator/Scenarios/ChedAManyCommoditiesScenarioGenerator.cs +++ b/TestDataGenerator/Scenarios/ChedAManyCommoditiesScenarioGenerator.cs @@ -9,7 +9,7 @@ public class ChedAManyCommoditiesScenarioGenerator(ILogger PreserveNewest - - + PreserveNewest - + PreserveNewest + + PreserveNewest +