CDMS-200 add test for movement not found (#23)

* CDMS-200 add test for movement not found CDMS-200 improved CachingBlobService implementation * CMDS-200 fixes tests * CDMS-200 refactors message processing * removes new test * Add decions in test generator and push to blob storage * Tidies up audit entry timings and source systems * Fixes up tests * Adds context to decisions * make audit entry version nullable * Handles checks in clearance request builder * Try to send clearance requests with the DecisionNumber set to the decision consumer * Updated TestDataGeneratorHelpers to use the bus to route to the consumers rather than call directly * tidy up the switch statement further * CDMS-200 adds ability to control sequencing of messages to builders * Switch Decisions to their own type * Switched back to using consumers directly * Finishes initial decision metrics * Tidied up code errors * Fixed up tests --------- Co-authored-by: Thomas Anderson <[email protected]>
DEFRA · Dec 18, 2024 · 5259079 · 5259079
1 parent 53326d0
commit 5259079
Show file tree

Hide file tree

Showing 70 changed files with 857 additions and 226 deletions.
diff --git a/Btms.Analytics.Tests/Btms.Analytics.Tests.csproj b/Btms.Analytics.Tests/Btms.Analytics.Tests.csproj
@@ -20,6 +20,7 @@
       <PackageReference Include="FluentAssertions" Version="6.12.1" />
       <PackageReference Include="MartinCostello.Logging.XUnit" Version="0.4.0" />
       <PackageReference Include="Microsoft.AspNetCore.Mvc.Testing" Version="8.0.8" />
+      <PackageReference Include="Microsoft.Extensions.Hosting.Abstractions" Version="9.0.0" />
       <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.11.1" />
       <PackageReference Include="SlimMessageBus" Version="2.0.4" />
       <PackageReference Include="xunit" Version="2.9.2" />

diff --git a/Btms.Analytics.Tests/Fixtures/BasicSampleDataTestFixture.cs b/Btms.Analytics.Tests/Fixtures/BasicSampleDataTestFixture.cs
@@ -2,7 +2,9 @@
 using Btms.Backend.Data;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging;
 using TestDataGenerator.Scenarios;
+using Xunit.Abstractions;
 
 namespace Btms.Analytics.Tests.Fixtures;
 
@@ -11,53 +13,65 @@ public class BasicSampleDataTestFixture : IDisposable
 #pragma warning restore S3881
 {
     public IHost App;
-    public IImportNotificationsAggregationService ImportNotificationsAggregationService;
-    public IMovementsAggregationService MovementsAggregationService;
-
-    public IMongoDbContext MongoDbContext;
-    public BasicSampleDataTestFixture()
+
+    private readonly IMongoDbContext _mongoDbContext;
+    private readonly ILogger<MultiItemDataTestFixture> _logger;
+    public BasicSampleDataTestFixture(IMessageSink messageSink)
     {
+        _logger = messageSink.ToLogger<MultiItemDataTestFixture>();
+
         var builder = TestContextHelper.CreateBuilder<BasicSampleDataTestFixture>();
 
         App = builder.Build();
         var rootScope = App.Services.CreateScope();
 
-        MongoDbContext = rootScope.ServiceProvider.GetRequiredService<IMongoDbContext>();
-        ImportNotificationsAggregationService = rootScope.ServiceProvider.GetRequiredService<IImportNotificationsAggregationService>();
-        MovementsAggregationService = rootScope.ServiceProvider.GetRequiredService<IMovementsAggregationService>();
+        _mongoDbContext = rootScope.ServiceProvider.GetRequiredService<IMongoDbContext>();
 
         // Would like to pick this up from env/config/DB state
         var insertToMongo = true;
 
         if (insertToMongo)
         {
-            MongoDbContext.ResetCollections().GetAwaiter().GetResult();
+            _mongoDbContext.ResetCollections().GetAwaiter().GetResult();
 
             // Ensure we have some data scenarios around 24/48 hour tests
-            App.PushToConsumers(App.CreateScenarioConfig<ChedASimpleMatchScenarioGenerator>(10, 3, arrivalDateRange: 0))
+            App.PushToConsumers(_logger, App.CreateScenarioConfig<ChedASimpleMatchScenarioGenerator>(10, 3, arrivalDateRange: 0))
                 .GetAwaiter().GetResult();
 
-            App.PushToConsumers(App.CreateScenarioConfig<ChedPSimpleMatchScenarioGenerator>(10, 3, arrivalDateRange: 2))
+            App.PushToConsumers(_logger, App.CreateScenarioConfig<ChedPSimpleMatchScenarioGenerator>(10, 3, arrivalDateRange: 2))
                 .GetAwaiter().GetResult();
 
-            App.PushToConsumers(App.CreateScenarioConfig<CrNoMatchScenarioGenerator>(10, 3, arrivalDateRange: 0))
+            App.PushToConsumers(_logger, App.CreateScenarioConfig<CrNoMatchScenarioGenerator>(10, 3, arrivalDateRange: 0))
                 .GetAwaiter().GetResult();
 
             // Create some more variable data over the rest of time
-            App.PushToConsumers(
+            App.PushToConsumers(_logger, 
                     App.CreateScenarioConfig<ChedASimpleMatchScenarioGenerator>(10, 7, arrivalDateRange: 10))
                 .GetAwaiter().GetResult();
 
-            App.PushToConsumers(App.CreateScenarioConfig<ChedANoMatchScenarioGenerator>(5, 3, arrivalDateRange: 10))
+            App.PushToConsumers(_logger, App.CreateScenarioConfig<ChedANoMatchScenarioGenerator>(5, 3, arrivalDateRange: 10))
                 .GetAwaiter().GetResult();
 
-            App.PushToConsumers(App.CreateScenarioConfig<ChedPSimpleMatchScenarioGenerator>(1, 3, arrivalDateRange: 10))
+            App.PushToConsumers(_logger, App.CreateScenarioConfig<ChedPSimpleMatchScenarioGenerator>(1, 3, arrivalDateRange: 10))
                 .GetAwaiter().GetResult();
 
-            App.PushToConsumers(App.CreateScenarioConfig<CrNoMatchScenarioGenerator>(1, 3, arrivalDateRange: 10))
+            App.PushToConsumers(_logger, App.CreateScenarioConfig<CrNoMatchScenarioGenerator>(1, 3, arrivalDateRange: 10))
                 .GetAwaiter().GetResult();
         }
     }
+
+
+    public IImportNotificationsAggregationService GetImportNotificationsAggregationService(ITestOutputHelper testOutputHelper)
+    {
+        var logger = testOutputHelper.GetLogger<ImportNotificationsAggregationService>();
+        return new ImportNotificationsAggregationService(_mongoDbContext, logger);   
+    }
+
+    public IMovementsAggregationService GetMovementsAggregationService(ITestOutputHelper testOutputHelper)
+    {
+        var logger = testOutputHelper.GetLogger<MovementsAggregationService>();
+        return new MovementsAggregationService(_mongoDbContext, logger);
+    }
 
     public void Dispose()
     {

diff --git a/Btms.Analytics.Tests/Fixtures/MultiItemDataTestFixture.cs b/Btms.Analytics.Tests/Fixtures/MultiItemDataTestFixture.cs
@@ -1,28 +1,30 @@
 using Btms.Analytics.Tests.Helpers;
 using Btms.Backend.Data;
 using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging;
 using TestDataGenerator.Scenarios;
+using Xunit.Abstractions;
 
 namespace Btms.Analytics.Tests.Fixtures;
 
 #pragma warning disable S3881
 public class MultiItemDataTestFixture : IDisposable
 #pragma warning restore S3881
 {
-    public readonly IImportNotificationsAggregationService ImportNotificationsAggregationService;
-    public readonly IMovementsAggregationService MovementsAggregationService;
+    public readonly IMongoDbContext MongoDbContext;
+    private readonly IServiceScope _rootScope;
+    private readonly ILogger<MultiItemDataTestFixture> _logger;
 
-    public IMongoDbContext MongoDbContext;
-    public MultiItemDataTestFixture()
+    public MultiItemDataTestFixture(IMessageSink messageSink)
     {
+        _logger = messageSink.ToLogger<MultiItemDataTestFixture>();
+
         var builder = TestContextHelper.CreateBuilder<MultiItemDataTestFixture>();
 
         var app = builder.Build();
-        var rootScope = app.Services.CreateScope();
+        _rootScope = app.Services.CreateScope();
 
-        MongoDbContext = rootScope.ServiceProvider.GetRequiredService<IMongoDbContext>();
-        ImportNotificationsAggregationService = rootScope.ServiceProvider.GetRequiredService<IImportNotificationsAggregationService>();
-        MovementsAggregationService = rootScope.ServiceProvider.GetRequiredService<IMovementsAggregationService>();
+        MongoDbContext = _rootScope.ServiceProvider.GetRequiredService<IMongoDbContext>();
 
         // Would like to pick this up from env/config/DB state
         var insertToMongo = true;
@@ -31,16 +33,33 @@ public MultiItemDataTestFixture()
         {
             MongoDbContext.ResetCollections().GetAwaiter().GetResult();
 
-            app.PushToConsumers(app.CreateScenarioConfig<ChedAManyCommoditiesScenarioGenerator>(10, 3, arrivalDateRange: 0))
+            app.PushToConsumers(_logger, app.CreateScenarioConfig<ChedAManyCommoditiesScenarioGenerator>(10, 3, arrivalDateRange: 0))
                 .GetAwaiter().GetResult();
-
-            app.PushToConsumers(app.CreateScenarioConfig<CrNoMatchScenarioGenerator>(10, 3, arrivalDateRange: 0))
+            
+            app.PushToConsumers(_logger, app.CreateScenarioConfig<CrNoMatchScenarioGenerator>(10, 3, arrivalDateRange: 0))
                 .GetAwaiter().GetResult();
-
-            app.PushToConsumers(app.CreateScenarioConfig<ChedASimpleMatchScenarioGenerator>(10, 3, arrivalDateRange: 0))
+
+            app.PushToConsumers(_logger, app.CreateScenarioConfig<ChedASimpleMatchScenarioGenerator>(10, 3, arrivalDateRange: 0))
+                .GetAwaiter().GetResult();
+
+            app.PushToConsumers(_logger, app.CreateScenarioConfig<ChedPSimpleMatchScenarioGenerator>(1, 1, arrivalDateRange: 0))
                 .GetAwaiter().GetResult();
         }
     }
+
+    public IImportNotificationsAggregationService GetImportNotificationsAggregationService(ITestOutputHelper testOutputHelper)
+    {
+        var logger = testOutputHelper.GetLogger<ImportNotificationsAggregationService>();
+        return new ImportNotificationsAggregationService(MongoDbContext, logger);   
+    }
+
+    public IMovementsAggregationService GetMovementsAggregationService(ITestOutputHelper testOutputHelper)
+    {
+        var logger = testOutputHelper.GetLogger<MovementsAggregationService>();
+        // return _rootScope.ServiceProvider.GetRequiredService<IMovementsAggregationService>(); 
+        return new MovementsAggregationService(MongoDbContext, logger);
+    }
+
 
     public void Dispose()
     {

diff --git a/Btms.Analytics.Tests/Helpers/ITestOutputHelperExtensions.cs b/Btms.Analytics.Tests/Helpers/ITestOutputHelperExtensions.cs
@@ -0,0 +1,16 @@
+using MartinCostello.Logging.XUnit;
+using Microsoft.Extensions.Logging;
+using Xunit.Abstractions;
+
+namespace Btms.Analytics.Tests.Helpers;
+
+public static class ITestOutputHelperExtensions
+{
+    public static ILogger<T> GetLogger<T>(this ITestOutputHelper helper)
+    {
+        var loggerProvider = new XUnitLoggerProvider(helper, new XUnitLoggerOptions());
+        var factory = new LoggerFactory([loggerProvider]);
+
+        return factory.CreateLogger<T>();
+    }
+}
diff --git a/Btms.Analytics.Tests/Helpers/TestDataGeneratorHelpers.cs b/Btms.Analytics.Tests/Helpers/TestDataGeneratorHelpers.cs
@@ -1,62 +1,98 @@
+using Btms.Common.Extensions;
 using Btms.Consumers;
 using Btms.Types.Alvs;
 using Btms.Types.Ipaffs;
+using Btms.SyncJob.Extensions;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.Logging;
 using SlimMessageBus;
 using SlimMessageBus.Host;
 using TestDataGenerator;
 using TestDataGenerator.Scenarios;
+using Decision = Btms.Types.Alvs.Decision;
 
 namespace Btms.Analytics.Tests.Helpers;
 
 public static class TestDataGeneratorHelpers
 {
-    private static int _scenarioIndex;
+    private static int scenarioIndex;
 
-    public static async Task<IHost> PushToConsumers(this IHost app, ScenarioConfig scenario)
+    public static async Task<IHost> PushToConsumers(this IHost app, ILogger logger, ScenarioConfig scenario)
     {
-        var generatorResults = app.Generate(scenario);
-        _scenarioIndex++;
+        var generatorResults = app.Generate(logger, scenario);
+        scenarioIndex++;
 
-        app.Services.GetRequiredService<ILogger<ScenarioGenerator>>();
+        // var logger = app.Services.GetRequiredService<ILogger<ScenarioGenerator>>();
+        var bus = app.Services.GetRequiredService<IPublishBus>();
 
         foreach (var generatorResult in generatorResults)
         {
-            foreach (var cr in generatorResult.ClearanceRequests)
+            foreach (var message in generatorResult)
             {
                 var scope = app.Services.CreateScope();
-                var consumer = (AlvsClearanceRequestConsumer)scope.ServiceProvider.GetRequiredService<IConsumer<AlvsClearanceRequest>>();
-
-                consumer.Context = new ConsumerContext
+
+                switch (message)
                 {
-                    Headers = new Dictionary<string, object> { { "messageId", cr.Header!.EntryReference! } }
-                };
-
-                await consumer.OnHandle(cr);
-            }
-
-            foreach (var n in generatorResult.ImportNotifications)
-            {
-                var scope = app.Services.CreateScope();
-                var consumer = (NotificationConsumer)scope.ServiceProvider.GetRequiredService<IConsumer<ImportNotification>>();
-
-                consumer.Context = new ConsumerContext
-                {
-                    Headers = new Dictionary<string, object> { { "messageId", n.ReferenceNumber! } }
-                };
-
-                await consumer.OnHandle(n);
+                    case null:
+                        throw new ArgumentNullException();
+
+                    case ImportNotification n:
+
+                        var notificationConsumer = (NotificationConsumer)scope
+                            .ServiceProvider
+                            .GetRequiredService<IConsumer<ImportNotification>>();
+
+                        notificationConsumer.Context = new ConsumerContext
+                        {
+                            Headers = new Dictionary<string, object> { { "messageId", n.ReferenceNumber! } }
+                        };
+
+                        await notificationConsumer.OnHandle(n);
+                        logger.LogInformation("Sent notification {0} to consumer", n.ReferenceNumber!);
+                        break;
+
+                    case Decision d:
+
+                        var decisionConsumer = (DecisionsConsumer)scope
+                            .ServiceProvider
+                            .GetRequiredService<IConsumer<Decision>>();
+
+                        decisionConsumer.Context = new ConsumerContext
+                        {
+                            Headers = new Dictionary<string, object> { { "messageId", d.Header!.EntryReference! } }
+                        };
+
+                        await decisionConsumer.OnHandle(d);
+                        logger.LogInformation("Sent decision {0} to consumer", d.Header!.EntryReference!);
+                        break;
+
+                    case AlvsClearanceRequest cr:
+
+                        var crConsumer = (AlvsClearanceRequestConsumer)scope
+                            .ServiceProvider
+                            .GetRequiredService<IConsumer<AlvsClearanceRequest>>();
+
+                        crConsumer.Context = new ConsumerContext
+                        {
+                            Headers = new Dictionary<string, object> { { "messageId", cr.Header!.EntryReference! } }
+                        };
+
+                        await crConsumer.OnHandle(cr);
+                        logger.LogInformation("Semt cr {0} to consumer", cr.Header!.EntryReference!);
+                        break;
+
+                    default:
+                        throw new ArgumentException($"Unexpected type {message.GetType().Name}");
+                }
             }
         }
 
         return app;
     } 
 
-    private static ScenarioGenerator.GeneratorResult[] Generate(this IHost app, ScenarioConfig scenario)
+    private static ScenarioGenerator.GeneratorResult[] Generate(this IHost app, ILogger logger, ScenarioConfig scenario)
     {
-        var logger = app.Services.GetRequiredService<ILogger<ScenarioGenerator>>();
         var days = scenario.CreationDateRange;
         var count = scenario.Count;
         var generator = scenario.Generator;
@@ -73,7 +109,7 @@ private static ScenarioGenerator.GeneratorResult[] Generate(this IHost app, Scen
             {
                 logger.LogInformation("Generating item {I}", i);
 
-                results.Add(generator.Generate(_scenarioIndex, i, entryDate, scenario));
+                results.Add(generator.Generate(scenarioIndex, i, entryDate, scenario));
             }
         }
 

diff --git a/Btms.Analytics.Tests/ImportNotificationsByArrivalDateTests.cs b/Btms.Analytics.Tests/ImportNotificationsByArrivalDateTests.cs
@@ -18,7 +18,7 @@ public async Task WhenCalledNextMonth_ReturnExpectedAggregation()
     {
         testOutputHelper.WriteLine("Querying for aggregated data");
 
-        var result = (await basicSampleDataTestFixture.ImportNotificationsAggregationService
+        var result = (await basicSampleDataTestFixture.GetImportNotificationsAggregationService(testOutputHelper)
             .ByArrival(DateTime.Today, DateTime.Today.MonthLater()))
             .Series
             .ToList();

diff --git a/Btms.Analytics.Tests/ImportNotificationsByCommoditiesTests.cs b/Btms.Analytics.Tests/ImportNotificationsByCommoditiesTests.cs
@@ -18,7 +18,7 @@ public class ImportNotificationsByCommoditiesTests(
     public async Task WhenCalledLastWeek_ReturnExpectedAggregation()
     {
         testOutputHelper.WriteLine("Querying for aggregated data");
-        var result = (await multiItemDataTestFixture.ImportNotificationsAggregationService
+        var result = (await multiItemDataTestFixture.GetImportNotificationsAggregationService(testOutputHelper)
             .ByCommodityCount(DateTime.Today.WeekAgo(), DateTime.Today.Tomorrow()))
             .Series
             .ToList();

diff --git a/Btms.Analytics.Tests/ImportNotificationsByCreatedDateTests.cs b/Btms.Analytics.Tests/ImportNotificationsByCreatedDateTests.cs
@@ -16,7 +16,7 @@ public class ImportNotificationsByCreatedDateTests(
     [Fact]
     public async Task WhenCalledLast48Hours_ReturnExpectedAggregation()
     {
-        var result = (await basicSampleDataTestFixture.ImportNotificationsAggregationService
+        var result = (await basicSampleDataTestFixture.GetImportNotificationsAggregationService(testOutputHelper)
             .ByCreated(DateTime.Now.NextHour().AddDays(-2), DateTime.Now.NextHour(),AggregationPeriod.Hour))
             .Series
             .ToList();
@@ -35,7 +35,7 @@ public async Task WhenCalledLast48Hours_ReturnExpectedAggregation()
     [Fact]
     public async Task WhenCalledLastMonth_ReturnExpectedAggregation()
     {
-        var result = (await basicSampleDataTestFixture.ImportNotificationsAggregationService
+        var result = (await basicSampleDataTestFixture.GetImportNotificationsAggregationService(testOutputHelper)
             .ByCreated(DateTime.Today.MonthAgo(), DateTime.Today.Tomorrow()))
             .Series
             .ToList();
@@ -63,7 +63,7 @@ public async Task WhenCalledWithTimePeriodYieldingNoResults_ReturnEmptyAggregati
         var from = DateTime.MaxValue.AddDays(-1);
         var to = DateTime.MaxValue;
 
-        var result = (await basicSampleDataTestFixture.ImportNotificationsAggregationService
+        var result = (await basicSampleDataTestFixture.GetImportNotificationsAggregationService(testOutputHelper)
                 .ByCreated(from, to, AggregationPeriod.Hour))
             .Series
             .ToList();