diff --git a/Btms.Analytics.Tests/Btms.Analytics.Tests.csproj b/Btms.Analytics.Tests/Btms.Analytics.Tests.csproj
index a246796..b612975 100644
--- a/Btms.Analytics.Tests/Btms.Analytics.Tests.csproj
+++ b/Btms.Analytics.Tests/Btms.Analytics.Tests.csproj
@@ -20,6 +20,7 @@
+
diff --git a/Btms.Analytics.Tests/Fixtures/BasicSampleDataTestFixture.cs b/Btms.Analytics.Tests/Fixtures/BasicSampleDataTestFixture.cs
index 5ee8684..312e43d 100644
--- a/Btms.Analytics.Tests/Fixtures/BasicSampleDataTestFixture.cs
+++ b/Btms.Analytics.Tests/Fixtures/BasicSampleDataTestFixture.cs
@@ -2,7 +2,9 @@
using Btms.Backend.Data;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging;
using TestDataGenerator.Scenarios;
+using Xunit.Abstractions;
namespace Btms.Analytics.Tests.Fixtures;
@@ -11,53 +13,65 @@ public class BasicSampleDataTestFixture : IDisposable
#pragma warning restore S3881
{
public IHost App;
- public IImportNotificationsAggregationService ImportNotificationsAggregationService;
- public IMovementsAggregationService MovementsAggregationService;
-
- public IMongoDbContext MongoDbContext;
- public BasicSampleDataTestFixture()
+
+ private readonly IMongoDbContext _mongoDbContext;
+ private readonly ILogger _logger;
+ public BasicSampleDataTestFixture(IMessageSink messageSink)
{
+ _logger = messageSink.ToLogger();
+
var builder = TestContextHelper.CreateBuilder();
App = builder.Build();
var rootScope = App.Services.CreateScope();
- MongoDbContext = rootScope.ServiceProvider.GetRequiredService();
- ImportNotificationsAggregationService = rootScope.ServiceProvider.GetRequiredService();
- MovementsAggregationService = rootScope.ServiceProvider.GetRequiredService();
+ _mongoDbContext = rootScope.ServiceProvider.GetRequiredService();
// Would like to pick this up from env/config/DB state
var insertToMongo = true;
if (insertToMongo)
{
- MongoDbContext.ResetCollections().GetAwaiter().GetResult();
+ _mongoDbContext.ResetCollections().GetAwaiter().GetResult();
// Ensure we have some data scenarios around 24/48 hour tests
- App.PushToConsumers(App.CreateScenarioConfig(10, 3, arrivalDateRange: 0))
+ App.PushToConsumers(_logger, App.CreateScenarioConfig(10, 3, arrivalDateRange: 0))
.GetAwaiter().GetResult();
- App.PushToConsumers(App.CreateScenarioConfig(10, 3, arrivalDateRange: 2))
+ App.PushToConsumers(_logger, App.CreateScenarioConfig(10, 3, arrivalDateRange: 2))
.GetAwaiter().GetResult();
- App.PushToConsumers(App.CreateScenarioConfig(10, 3, arrivalDateRange: 0))
+ App.PushToConsumers(_logger, App.CreateScenarioConfig(10, 3, arrivalDateRange: 0))
.GetAwaiter().GetResult();
// Create some more variable data over the rest of time
- App.PushToConsumers(
+ App.PushToConsumers(_logger,
App.CreateScenarioConfig(10, 7, arrivalDateRange: 10))
.GetAwaiter().GetResult();
- App.PushToConsumers(App.CreateScenarioConfig(5, 3, arrivalDateRange: 10))
+ App.PushToConsumers(_logger, App.CreateScenarioConfig(5, 3, arrivalDateRange: 10))
.GetAwaiter().GetResult();
- App.PushToConsumers(App.CreateScenarioConfig(1, 3, arrivalDateRange: 10))
+ App.PushToConsumers(_logger, App.CreateScenarioConfig(1, 3, arrivalDateRange: 10))
.GetAwaiter().GetResult();
- App.PushToConsumers(App.CreateScenarioConfig(1, 3, arrivalDateRange: 10))
+ App.PushToConsumers(_logger, App.CreateScenarioConfig(1, 3, arrivalDateRange: 10))
.GetAwaiter().GetResult();
}
}
+
+
+ public IImportNotificationsAggregationService GetImportNotificationsAggregationService(ITestOutputHelper testOutputHelper)
+ {
+ var logger = testOutputHelper.GetLogger();
+ return new ImportNotificationsAggregationService(_mongoDbContext, logger);
+ }
+
+ public IMovementsAggregationService GetMovementsAggregationService(ITestOutputHelper testOutputHelper)
+ {
+ var logger = testOutputHelper.GetLogger();
+ return new MovementsAggregationService(_mongoDbContext, logger);
+ }
public void Dispose()
{
diff --git a/Btms.Analytics.Tests/Fixtures/MultiItemDataTestFixture.cs b/Btms.Analytics.Tests/Fixtures/MultiItemDataTestFixture.cs
index c322101..f39295c 100644
--- a/Btms.Analytics.Tests/Fixtures/MultiItemDataTestFixture.cs
+++ b/Btms.Analytics.Tests/Fixtures/MultiItemDataTestFixture.cs
@@ -1,7 +1,9 @@
using Btms.Analytics.Tests.Helpers;
using Btms.Backend.Data;
using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging;
using TestDataGenerator.Scenarios;
+using Xunit.Abstractions;
namespace Btms.Analytics.Tests.Fixtures;
@@ -9,20 +11,20 @@ namespace Btms.Analytics.Tests.Fixtures;
public class MultiItemDataTestFixture : IDisposable
#pragma warning restore S3881
{
- public readonly IImportNotificationsAggregationService ImportNotificationsAggregationService;
- public readonly IMovementsAggregationService MovementsAggregationService;
+ public readonly IMongoDbContext MongoDbContext;
+ private readonly IServiceScope _rootScope;
+ private readonly ILogger _logger;
- public IMongoDbContext MongoDbContext;
- public MultiItemDataTestFixture()
+ public MultiItemDataTestFixture(IMessageSink messageSink)
{
+ _logger = messageSink.ToLogger();
+
var builder = TestContextHelper.CreateBuilder();
var app = builder.Build();
- var rootScope = app.Services.CreateScope();
+ _rootScope = app.Services.CreateScope();
- MongoDbContext = rootScope.ServiceProvider.GetRequiredService();
- ImportNotificationsAggregationService = rootScope.ServiceProvider.GetRequiredService();
- MovementsAggregationService = rootScope.ServiceProvider.GetRequiredService();
+ MongoDbContext = _rootScope.ServiceProvider.GetRequiredService();
// Would like to pick this up from env/config/DB state
var insertToMongo = true;
@@ -31,16 +33,33 @@ public MultiItemDataTestFixture()
{
MongoDbContext.ResetCollections().GetAwaiter().GetResult();
- app.PushToConsumers(app.CreateScenarioConfig(10, 3, arrivalDateRange: 0))
+ app.PushToConsumers(_logger, app.CreateScenarioConfig(10, 3, arrivalDateRange: 0))
.GetAwaiter().GetResult();
-
- app.PushToConsumers(app.CreateScenarioConfig(10, 3, arrivalDateRange: 0))
+
+ app.PushToConsumers(_logger, app.CreateScenarioConfig(10, 3, arrivalDateRange: 0))
.GetAwaiter().GetResult();
-
- app.PushToConsumers(app.CreateScenarioConfig(10, 3, arrivalDateRange: 0))
+
+ app.PushToConsumers(_logger, app.CreateScenarioConfig(10, 3, arrivalDateRange: 0))
+ .GetAwaiter().GetResult();
+
+ app.PushToConsumers(_logger, app.CreateScenarioConfig(1, 1, arrivalDateRange: 0))
.GetAwaiter().GetResult();
}
}
+
+ public IImportNotificationsAggregationService GetImportNotificationsAggregationService(ITestOutputHelper testOutputHelper)
+ {
+ var logger = testOutputHelper.GetLogger();
+ return new ImportNotificationsAggregationService(MongoDbContext, logger);
+ }
+
+ public IMovementsAggregationService GetMovementsAggregationService(ITestOutputHelper testOutputHelper)
+ {
+ var logger = testOutputHelper.GetLogger();
+ // return _rootScope.ServiceProvider.GetRequiredService();
+ return new MovementsAggregationService(MongoDbContext, logger);
+ }
+
public void Dispose()
{
diff --git a/Btms.Analytics.Tests/Helpers/ITestOutputHelperExtensions.cs b/Btms.Analytics.Tests/Helpers/ITestOutputHelperExtensions.cs
new file mode 100644
index 0000000..e495e10
--- /dev/null
+++ b/Btms.Analytics.Tests/Helpers/ITestOutputHelperExtensions.cs
@@ -0,0 +1,16 @@
+using MartinCostello.Logging.XUnit;
+using Microsoft.Extensions.Logging;
+using Xunit.Abstractions;
+
+namespace Btms.Analytics.Tests.Helpers;
+
+public static class ITestOutputHelperExtensions
+{
+ public static ILogger GetLogger(this ITestOutputHelper helper)
+ {
+ var loggerProvider = new XUnitLoggerProvider(helper, new XUnitLoggerOptions());
+ var factory = new LoggerFactory([loggerProvider]);
+
+ return factory.CreateLogger();
+ }
+}
\ No newline at end of file
diff --git a/Btms.Analytics.Tests/Helpers/TestDataGeneratorHelpers.cs b/Btms.Analytics.Tests/Helpers/TestDataGeneratorHelpers.cs
index d7343f2..aaba184 100644
--- a/Btms.Analytics.Tests/Helpers/TestDataGeneratorHelpers.cs
+++ b/Btms.Analytics.Tests/Helpers/TestDataGeneratorHelpers.cs
@@ -1,6 +1,8 @@
+using Btms.Common.Extensions;
using Btms.Consumers;
using Btms.Types.Alvs;
using Btms.Types.Ipaffs;
+using Btms.SyncJob.Extensions;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
@@ -8,55 +10,89 @@
using SlimMessageBus.Host;
using TestDataGenerator;
using TestDataGenerator.Scenarios;
+using Decision = Btms.Types.Alvs.Decision;
namespace Btms.Analytics.Tests.Helpers;
public static class TestDataGeneratorHelpers
{
- private static int _scenarioIndex;
+ private static int scenarioIndex;
- public static async Task PushToConsumers(this IHost app, ScenarioConfig scenario)
+ public static async Task PushToConsumers(this IHost app, ILogger logger, ScenarioConfig scenario)
{
- var generatorResults = app.Generate(scenario);
- _scenarioIndex++;
+ var generatorResults = app.Generate(logger, scenario);
+ scenarioIndex++;
- app.Services.GetRequiredService>();
+ // var logger = app.Services.GetRequiredService>();
+ var bus = app.Services.GetRequiredService();
foreach (var generatorResult in generatorResults)
{
- foreach (var cr in generatorResult.ClearanceRequests)
+ foreach (var message in generatorResult)
{
var scope = app.Services.CreateScope();
- var consumer = (AlvsClearanceRequestConsumer)scope.ServiceProvider.GetRequiredService>();
-
- consumer.Context = new ConsumerContext
+
+ switch (message)
{
- Headers = new Dictionary { { "messageId", cr.Header!.EntryReference! } }
- };
-
- await consumer.OnHandle(cr);
- }
-
- foreach (var n in generatorResult.ImportNotifications)
- {
- var scope = app.Services.CreateScope();
- var consumer = (NotificationConsumer)scope.ServiceProvider.GetRequiredService>();
-
- consumer.Context = new ConsumerContext
- {
- Headers = new Dictionary { { "messageId", n.ReferenceNumber! } }
- };
-
- await consumer.OnHandle(n);
+ case null:
+ throw new ArgumentNullException();
+
+ case ImportNotification n:
+
+ var notificationConsumer = (NotificationConsumer)scope
+ .ServiceProvider
+ .GetRequiredService>();
+
+ notificationConsumer.Context = new ConsumerContext
+ {
+ Headers = new Dictionary { { "messageId", n.ReferenceNumber! } }
+ };
+
+ await notificationConsumer.OnHandle(n);
+ logger.LogInformation("Sent notification {0} to consumer", n.ReferenceNumber!);
+ break;
+
+ case Decision d:
+
+ var decisionConsumer = (DecisionsConsumer)scope
+ .ServiceProvider
+ .GetRequiredService>();
+
+ decisionConsumer.Context = new ConsumerContext
+ {
+ Headers = new Dictionary { { "messageId", d.Header!.EntryReference! } }
+ };
+
+ await decisionConsumer.OnHandle(d);
+ logger.LogInformation("Sent decision {0} to consumer", d.Header!.EntryReference!);
+ break;
+
+ case AlvsClearanceRequest cr:
+
+ var crConsumer = (AlvsClearanceRequestConsumer)scope
+ .ServiceProvider
+ .GetRequiredService>();
+
+ crConsumer.Context = new ConsumerContext
+ {
+ Headers = new Dictionary { { "messageId", cr.Header!.EntryReference! } }
+ };
+
+ await crConsumer.OnHandle(cr);
+ logger.LogInformation("Semt cr {0} to consumer", cr.Header!.EntryReference!);
+ break;
+
+ default:
+ throw new ArgumentException($"Unexpected type {message.GetType().Name}");
+ }
}
}
return app;
}
- private static ScenarioGenerator.GeneratorResult[] Generate(this IHost app, ScenarioConfig scenario)
+ private static ScenarioGenerator.GeneratorResult[] Generate(this IHost app, ILogger logger, ScenarioConfig scenario)
{
- var logger = app.Services.GetRequiredService>();
var days = scenario.CreationDateRange;
var count = scenario.Count;
var generator = scenario.Generator;
@@ -73,7 +109,7 @@ private static ScenarioGenerator.GeneratorResult[] Generate(this IHost app, Scen
{
logger.LogInformation("Generating item {I}", i);
- results.Add(generator.Generate(_scenarioIndex, i, entryDate, scenario));
+ results.Add(generator.Generate(scenarioIndex, i, entryDate, scenario));
}
}
diff --git a/Btms.Analytics.Tests/ImportNotificationsByArrivalDateTests.cs b/Btms.Analytics.Tests/ImportNotificationsByArrivalDateTests.cs
index 61c9baf..9034a8f 100644
--- a/Btms.Analytics.Tests/ImportNotificationsByArrivalDateTests.cs
+++ b/Btms.Analytics.Tests/ImportNotificationsByArrivalDateTests.cs
@@ -18,7 +18,7 @@ public async Task WhenCalledNextMonth_ReturnExpectedAggregation()
{
testOutputHelper.WriteLine("Querying for aggregated data");
- var result = (await basicSampleDataTestFixture.ImportNotificationsAggregationService
+ var result = (await basicSampleDataTestFixture.GetImportNotificationsAggregationService(testOutputHelper)
.ByArrival(DateTime.Today, DateTime.Today.MonthLater()))
.Series
.ToList();
diff --git a/Btms.Analytics.Tests/ImportNotificationsByCommoditiesTests.cs b/Btms.Analytics.Tests/ImportNotificationsByCommoditiesTests.cs
index 10ce1dc..735ba3d 100644
--- a/Btms.Analytics.Tests/ImportNotificationsByCommoditiesTests.cs
+++ b/Btms.Analytics.Tests/ImportNotificationsByCommoditiesTests.cs
@@ -18,7 +18,7 @@ public class ImportNotificationsByCommoditiesTests(
public async Task WhenCalledLastWeek_ReturnExpectedAggregation()
{
testOutputHelper.WriteLine("Querying for aggregated data");
- var result = (await multiItemDataTestFixture.ImportNotificationsAggregationService
+ var result = (await multiItemDataTestFixture.GetImportNotificationsAggregationService(testOutputHelper)
.ByCommodityCount(DateTime.Today.WeekAgo(), DateTime.Today.Tomorrow()))
.Series
.ToList();
diff --git a/Btms.Analytics.Tests/ImportNotificationsByCreatedDateTests.cs b/Btms.Analytics.Tests/ImportNotificationsByCreatedDateTests.cs
index de4e3f1..a3ba540 100644
--- a/Btms.Analytics.Tests/ImportNotificationsByCreatedDateTests.cs
+++ b/Btms.Analytics.Tests/ImportNotificationsByCreatedDateTests.cs
@@ -16,7 +16,7 @@ public class ImportNotificationsByCreatedDateTests(
[Fact]
public async Task WhenCalledLast48Hours_ReturnExpectedAggregation()
{
- var result = (await basicSampleDataTestFixture.ImportNotificationsAggregationService
+ var result = (await basicSampleDataTestFixture.GetImportNotificationsAggregationService(testOutputHelper)
.ByCreated(DateTime.Now.NextHour().AddDays(-2), DateTime.Now.NextHour(),AggregationPeriod.Hour))
.Series
.ToList();
@@ -35,7 +35,7 @@ public async Task WhenCalledLast48Hours_ReturnExpectedAggregation()
[Fact]
public async Task WhenCalledLastMonth_ReturnExpectedAggregation()
{
- var result = (await basicSampleDataTestFixture.ImportNotificationsAggregationService
+ var result = (await basicSampleDataTestFixture.GetImportNotificationsAggregationService(testOutputHelper)
.ByCreated(DateTime.Today.MonthAgo(), DateTime.Today.Tomorrow()))
.Series
.ToList();
@@ -63,7 +63,7 @@ public async Task WhenCalledWithTimePeriodYieldingNoResults_ReturnEmptyAggregati
var from = DateTime.MaxValue.AddDays(-1);
var to = DateTime.MaxValue;
- var result = (await basicSampleDataTestFixture.ImportNotificationsAggregationService
+ var result = (await basicSampleDataTestFixture.GetImportNotificationsAggregationService(testOutputHelper)
.ByCreated(from, to, AggregationPeriod.Hour))
.Series
.ToList();
diff --git a/Btms.Analytics.Tests/ImportNotificationsByStatusTests.cs b/Btms.Analytics.Tests/ImportNotificationsByStatusTests.cs
index 8ad2745..07e3063 100644
--- a/Btms.Analytics.Tests/ImportNotificationsByStatusTests.cs
+++ b/Btms.Analytics.Tests/ImportNotificationsByStatusTests.cs
@@ -17,7 +17,7 @@ public class ImportNotificationsByStatusTests(
public async Task WhenCalledLastWeek_ReturnExpectedAggregation()
{
testOutputHelper.WriteLine("Querying for aggregated data");
- var result = (await basicSampleDataTestFixture.ImportNotificationsAggregationService
+ var result = (await basicSampleDataTestFixture.GetImportNotificationsAggregationService(testOutputHelper)
.ByStatus(DateTime.Today.WeekAgo(), DateTime.Today.Tomorrow()));
testOutputHelper.WriteLine("{0} aggregated items found", result.Values.Count);
@@ -29,7 +29,7 @@ public async Task WhenCalledLastWeek_ReturnExpectedAggregation()
public async Task WhenCalledLast48Hours_ReturnExpectedAggregation()
{
testOutputHelper.WriteLine("Querying for aggregated data");
- var result = (await basicSampleDataTestFixture.ImportNotificationsAggregationService
+ var result = (await basicSampleDataTestFixture.GetImportNotificationsAggregationService(testOutputHelper)
.ByStatus(DateTime.Now.NextHour().AddDays(-2), DateTime.Now.NextHour()));
testOutputHelper.WriteLine($"{result.Values.Count} aggregated items found");
@@ -44,7 +44,7 @@ public async Task WhenCalledLast48Hours_ReturnExpectedAggregation()
public async Task WhenCalledWithTimePeriodYieldingNoResults_ReturnEmptyAggregation()
{
testOutputHelper.WriteLine("Querying for aggregated data");
- var result = (await basicSampleDataTestFixture.ImportNotificationsAggregationService
+ var result = (await basicSampleDataTestFixture.GetImportNotificationsAggregationService(testOutputHelper)
.ByStatus(DateTime.MaxValue.AddDays(-1), DateTime.MaxValue));
testOutputHelper.WriteLine($"{result.Values.Count} aggregated items found");
diff --git a/Btms.Analytics.Tests/MovementHistoryTests.cs b/Btms.Analytics.Tests/MovementHistoryTests.cs
index 8f356f7..d8d62e9 100644
--- a/Btms.Analytics.Tests/MovementHistoryTests.cs
+++ b/Btms.Analytics.Tests/MovementHistoryTests.cs
@@ -14,15 +14,34 @@ public class MovementHistoryTests(
ITestOutputHelper testOutputHelper)
{
[Fact]
- public async Task WhenCalled_ReturnsHistory()
+ public async Task WhenCalledWithAMovementThatHasADecision_ReturnsHistory()
{
+ testOutputHelper.WriteLine("Find a suitable Movement with history");
+ var movement = await multiItemDataTestFixture.MongoDbContext.Movements.Find(
+ m =>
+ m.Relationships.Notifications.Data.Count > 0
+ && m.Decisions.Count > 0);
+
+ ArgumentNullException.ThrowIfNull(movement);
+
testOutputHelper.WriteLine("Querying for history");
- var result = await multiItemDataTestFixture.MovementsAggregationService
- .GetHistory("23GB9999001215000001");
+ var result = await multiItemDataTestFixture
+ .GetMovementsAggregationService(testOutputHelper)
+ .GetHistory(movement.Id!);
- testOutputHelper.WriteLine("{0} history items found", result.Items.Count());
+ testOutputHelper.WriteLine("{0} history items found", result!.Items.Count());
result.Items.Should().HasValue();
result.Items.Select(a => a.AuditEntry.CreatedSource).Should().BeInAscendingOrder();
}
+
+ [Fact]
+ public async Task WhenCalledWithAFakeMovementID_ReturnsNoHistory()
+ {
+ testOutputHelper.WriteLine("Querying for history");
+ var result = await multiItemDataTestFixture.GetMovementsAggregationService(testOutputHelper)
+ .GetHistory("");
+
+ result.Should().BeNull();
+ }
}
\ No newline at end of file
diff --git a/Btms.Analytics.Tests/MovementsByCreatedDateTests.cs b/Btms.Analytics.Tests/MovementsByCreatedDateTests.cs
index e0173b3..c90975f 100644
--- a/Btms.Analytics.Tests/MovementsByCreatedDateTests.cs
+++ b/Btms.Analytics.Tests/MovementsByCreatedDateTests.cs
@@ -15,7 +15,7 @@ public class MovementsByCreatedDateTests(
[Fact]
public async Task WhenCalledLast48Hours_ReturnExpectedAggregation()
{
- var result = (await basicSampleDataTestFixture.MovementsAggregationService
+ var result = (await basicSampleDataTestFixture.GetMovementsAggregationService(testOutputHelper)
.ByCreated(DateTime.Now.NextHour().AddDays(-2), DateTime.Now.NextHour(), AggregationPeriod.Hour))
.Series
.ToList();
@@ -37,7 +37,7 @@ public async Task WhenCalledWithTimePeriodYieldingNoResults_ReturnEmptyAggregati
var from = DateTime.MaxValue.AddDays(-1);
var to = DateTime.MaxValue;
- var result = (await basicSampleDataTestFixture.MovementsAggregationService
+ var result = (await basicSampleDataTestFixture.GetMovementsAggregationService(testOutputHelper)
.ByCreated(from, to, AggregationPeriod.Hour))
.Series
.ToList();
@@ -62,7 +62,7 @@ public async Task WhenCalledWithTimePeriodYieldingNoResults_ReturnEmptyAggregati
[Fact]
public async Task WhenCalledLastMonth_ReturnExpectedAggregation()
{
- var result = (await basicSampleDataTestFixture.MovementsAggregationService
+ var result = (await basicSampleDataTestFixture.GetMovementsAggregationService(testOutputHelper)
.ByCreated(DateTime.Today.MonthAgo(), DateTime.Today.Tomorrow()))
.Series
.ToList();
diff --git a/Btms.Analytics.Tests/MovementsByDecisionsTests.cs b/Btms.Analytics.Tests/MovementsByDecisionsTests.cs
new file mode 100644
index 0000000..ae215ee
--- /dev/null
+++ b/Btms.Analytics.Tests/MovementsByDecisionsTests.cs
@@ -0,0 +1,32 @@
+using Btms.Common.Extensions;
+using FluentAssertions;
+using Xunit;
+using Xunit.Abstractions;
+
+using Btms.Analytics.Tests.Fixtures;
+using Btms.Analytics.Tests.Helpers;
+
+namespace Btms.Analytics.Tests;
+
+[Collection(nameof(MultiItemDataTestCollection))]
+public class MovementsByDecisionsTests(
+ MultiItemDataTestFixture multiItemDataTestFixture,
+ ITestOutputHelper testOutputHelper)
+{
+
+ [Fact]
+ public async Task WhenCalled_ReturnExpectedAggregation()
+ {
+ testOutputHelper.WriteLine("Querying for aggregated data");
+ var result = (await multiItemDataTestFixture.GetMovementsAggregationService(testOutputHelper)
+ .ByDecision(DateTime.Today.MonthAgo(), DateTime.Today.Tomorrow()))
+ .Values
+ .ToList();
+
+ testOutputHelper.WriteLine("{0} aggregated items found", result.Count);
+
+ result.Count.Should().Be(3);
+ result.Select(r => r.Key).Order().Should()
+ .Equal("ALVS Linked : H01", "BTMS Linked : X00", "BTMS Not Linked : X00");
+ }
+}
\ No newline at end of file
diff --git a/Btms.Analytics.Tests/MovementsByItemsTests.cs b/Btms.Analytics.Tests/MovementsByItemsTests.cs
index 477367a..0d82f73 100644
--- a/Btms.Analytics.Tests/MovementsByItemsTests.cs
+++ b/Btms.Analytics.Tests/MovementsByItemsTests.cs
@@ -18,7 +18,7 @@ public class MovementsByItemsTests(
public async Task WhenCalledLastWeek_ReturnExpectedAggregation()
{
testOutputHelper.WriteLine("Querying for aggregated data");
- var result = (await multiItemDataTestFixture.MovementsAggregationService
+ var result = (await multiItemDataTestFixture.GetMovementsAggregationService(testOutputHelper)
.ByItemCount(DateTime.Today.WeekAgo(), DateTime.Today.Tomorrow()))
.Series
.ToList();
diff --git a/Btms.Analytics.Tests/MovementsByStatusTests.cs b/Btms.Analytics.Tests/MovementsByStatusTests.cs
index 006ab2f..a66f0a4 100644
--- a/Btms.Analytics.Tests/MovementsByStatusTests.cs
+++ b/Btms.Analytics.Tests/MovementsByStatusTests.cs
@@ -16,7 +16,7 @@ public class MovementsByStatusTests(
public async Task WhenCalledLastWeek_ReturnExpectedAggregation()
{
testOutputHelper.WriteLine("Querying for aggregated data");
- var result = (await basicSampleDataTestFixture.MovementsAggregationService
+ var result = (await basicSampleDataTestFixture.GetMovementsAggregationService(testOutputHelper)
.ByStatus(DateTime.Today.WeekAgo(), DateTime.Today.Tomorrow()));
testOutputHelper.WriteLine("{0} aggregated items found", result.Values.Count);
@@ -29,7 +29,7 @@ public async Task WhenCalledLastWeek_ReturnExpectedAggregation()
public async Task WhenCalledLast48Hours_ReturnExpectedAggregation()
{
testOutputHelper.WriteLine("Querying for aggregated data");
- var result = (await basicSampleDataTestFixture.MovementsAggregationService
+ var result = (await basicSampleDataTestFixture.GetMovementsAggregationService(testOutputHelper)
.ByStatus(DateTime.Now.NextHour().AddDays(-2), DateTime.Now.NextHour()));
testOutputHelper.WriteLine($"{result.Values.Count} aggregated items found");
@@ -42,7 +42,7 @@ public async Task WhenCalledLast48Hours_ReturnExpectedAggregation()
public async Task WhenCalledWithTimePeriodYieldingNoResults_ReturnEmptyAggregation()
{
testOutputHelper.WriteLine("Querying for aggregated data");
- var result = (await basicSampleDataTestFixture.MovementsAggregationService
+ var result = (await basicSampleDataTestFixture.GetMovementsAggregationService(testOutputHelper)
.ByStatus(DateTime.MaxValue.AddDays(-1), DateTime.MaxValue));
testOutputHelper.WriteLine($"{result.Values.Count} aggregated items found");
diff --git a/Btms.Analytics.Tests/MovementsByUniqueDocumentReferenceTests.cs b/Btms.Analytics.Tests/MovementsByUniqueDocumentReferenceTests.cs
index 52c61f3..f7a17e2 100644
--- a/Btms.Analytics.Tests/MovementsByUniqueDocumentReferenceTests.cs
+++ b/Btms.Analytics.Tests/MovementsByUniqueDocumentReferenceTests.cs
@@ -18,7 +18,7 @@ public class MovementsByUniqueDocumentReferenceTests(
public async Task WhenCalledLastWeek_ReturnExpectedAggregation()
{
testOutputHelper.WriteLine("Querying for aggregated data");
- var result = (await multiItemDataTestFixture.MovementsAggregationService
+ var result = (await multiItemDataTestFixture.GetMovementsAggregationService(testOutputHelper)
.ByUniqueDocumentReferenceCount(DateTime.Today.WeekAgo(), DateTime.Today.Tomorrow()))
.Series
.ToList();
diff --git a/Btms.Analytics.Tests/MovementsDocumentReferencesByMovementTests.cs b/Btms.Analytics.Tests/MovementsDocumentReferencesByMovementTests.cs
index 6a2ed05..4de74ec 100644
--- a/Btms.Analytics.Tests/MovementsDocumentReferencesByMovementTests.cs
+++ b/Btms.Analytics.Tests/MovementsDocumentReferencesByMovementTests.cs
@@ -16,7 +16,7 @@ public class MovementsDocumentReferencesByMovementTests(
public async Task WhenCalledLastWeek_ReturnExpectedAggregation()
{
testOutputHelper.WriteLine("Querying for aggregated data");
- var result = (await multiItemDataTestFixture.MovementsAggregationService
+ var result = (await multiItemDataTestFixture.GetMovementsAggregationService(testOutputHelper)
.UniqueDocumentReferenceByMovementCount(DateTime.Today.WeekAgo(), DateTime.Today.Tomorrow()));
testOutputHelper.WriteLine("{0} aggregated items found", result.Values.Count);
diff --git a/Btms.Analytics/IMovementsAggregationService.cs b/Btms.Analytics/IMovementsAggregationService.cs
index cd65eda..948c8ec 100644
--- a/Btms.Analytics/IMovementsAggregationService.cs
+++ b/Btms.Analytics/IMovementsAggregationService.cs
@@ -7,8 +7,9 @@ public interface IMovementsAggregationService
public Task ByCreated(DateTime from, DateTime to, AggregationPeriod aggregateBy = AggregationPeriod.Day);
public Task ByStatus(DateTime from, DateTime to);
public Task ByItemCount(DateTime from, DateTime to);
+ public Task ByDecision(DateTime from, DateTime to);
public Task ByUniqueDocumentReferenceCount(DateTime from, DateTime to);
public Task UniqueDocumentReferenceByMovementCount(DateTime from, DateTime to);
public Task ByCheck(DateTime from, DateTime to);
- public Task> GetHistory(string movementId);
+ public Task?> GetHistory(string movementId);
}
\ No newline at end of file
diff --git a/Btms.Analytics/MovementsAggregationService.cs b/Btms.Analytics/MovementsAggregationService.cs
index 7ce5169..9b14f6a 100644
--- a/Btms.Analytics/MovementsAggregationService.cs
+++ b/Btms.Analytics/MovementsAggregationService.cs
@@ -2,6 +2,7 @@
using System.Linq.Expressions;
using Btms.Analytics.Extensions;
using Btms.Backend.Data;
+using Btms.Common.Extensions;
using Btms.Model.Extensions;
using Btms.Model;
using Btms.Model.Auditing;
@@ -148,12 +149,17 @@ public Task UniqueDocumentReferenceByMovementCount(DateTime
return Task.FromResult(result);
}
- public async Task> GetHistory(string movementId)
+ public async Task?> GetHistory(string movementId)
{
var movement = await context
.Movements
.Find(movementId);
+ if (!movement.HasValue())
+ {
+ return null;
+ }
+
var notificationIds = movement!.Relationships.Notifications.Data.Select(n => n.Id);
var notificationEntries = context.Notifications
@@ -202,4 +208,41 @@ private Task Aggregate(DateTime[] dateRange, Func ByDecision(DateTime from, DateTime to)
+ {
+ var mongoQuery = context
+ .Movements
+ .Where(m => m.CreatedSource >= from && m.CreatedSource < to)
+ .SelectMany(m => m.Decisions.Select(d => new { Decision = d, Movement = m }))
+ .SelectMany(m =>
+ m.Decision.Items!.Select(i => new { Decision = m.Decision, Movement = m.Movement, Item = i }))
+ .SelectMany(m => m.Item.Checks!.Select(c => new
+ {
+ CheckCode = c.CheckCode,
+ DecisionCode = c.DecisionCode,
+ DecisionSourceSystem = m.Decision.ServiceHeader!.SourceSystem,
+ DecisionEntryReference = m.Decision.Header!.EntryReference,
+ DecisionEntryVersionNumber = m.Decision.Header!.EntryVersionNumber,
+ Movement = m.Movement.EntryReference,
+ MovementVersion = m.Movement.EntryVersionNumber,
+ HasLinks = m.Movement.Relationships.Notifications.Data.Count > 0,
+ ItemNumber = m.Item.ItemNumber
+ }))
+ .GroupBy(m => new { m.HasLinks, m.DecisionSourceSystem, m.DecisionCode })
+ .Select(m => new { m.Key.HasLinks, m.Key.DecisionSourceSystem, m.Key.DecisionCode, Count = m.Count() })
+ .ToList();
+
+ logger.LogInformation("Found {0} items", mongoQuery.Count);
+ logger.LogInformation(mongoQuery.ToJsonString());
+
+ return Task.FromResult(new SingleSeriesDataset()
+ {
+ Values = mongoQuery
+ .ToDictionary(
+ r => $"{ r.DecisionSourceSystem } { ( r.HasLinks ? "Linked" : "Not Linked" ) } : { r.DecisionCode }",
+ r => r.Count
+ )
+ });
+ }
}
\ No newline at end of file
diff --git a/Btms.Backend.Data/IMongoCollectionSet.cs b/Btms.Backend.Data/IMongoCollectionSet.cs
index 29a8967..164f8ab 100644
--- a/Btms.Backend.Data/IMongoCollectionSet.cs
+++ b/Btms.Backend.Data/IMongoCollectionSet.cs
@@ -1,3 +1,4 @@
+using System.Linq.Expressions;
using Btms.Model.Data;
using MongoDB.Driver;
@@ -6,6 +7,8 @@ namespace Btms.Backend.Data;
public interface IMongoCollectionSet : IQueryable where T : IDataEntity
{
Task Find(string id);
+ Task Find(Expression> query);
+
Task Insert(T item, IMongoDbTransaction transaction = default!, CancellationToken cancellationToken = default);
Task Update(T item, string etag, IMongoDbTransaction transaction = default!,
diff --git a/Btms.Backend.Data/InMemory/MemoryCollectionSet.cs b/Btms.Backend.Data/InMemory/MemoryCollectionSet.cs
index 913c804..36295af 100644
--- a/Btms.Backend.Data/InMemory/MemoryCollectionSet.cs
+++ b/Btms.Backend.Data/InMemory/MemoryCollectionSet.cs
@@ -31,6 +31,11 @@ IEnumerator IEnumerable.GetEnumerator()
return Task.FromResult(data.Find(x => x.Id == id));
}
+ public Task Find(Expression> query)
+ {
+ return Task.FromResult(data.Find(i => query.Compile()(i)));
+ }
+
public Task Insert(T item, IMongoDbTransaction transaction = default!, CancellationToken cancellationToken = default)
{
item._Etag = BsonObjectIdGenerator.Instance.GenerateId(null, null).ToString()!;
diff --git a/Btms.Backend.Data/Mongo/MongoCollectionSet.cs b/Btms.Backend.Data/Mongo/MongoCollectionSet.cs
index 6490ec2..a8954f7 100644
--- a/Btms.Backend.Data/Mongo/MongoCollectionSet.cs
+++ b/Btms.Backend.Data/Mongo/MongoCollectionSet.cs
@@ -4,6 +4,7 @@
using MongoDB.Driver.Linq;
using System.Collections;
using System.Linq.Expressions;
+using Microsoft.EntityFrameworkCore;
namespace Btms.Backend.Data.Mongo;
@@ -35,6 +36,11 @@ IEnumerator IEnumerable.GetEnumerator()
return await EntityQueryable.SingleOrDefaultAsync(x => x.Id == id);
}
+ public async Task Find(Expression> query)
+ {
+ return await EntityQueryable.FirstOrDefaultAsync(query);
+ }
+
public Task Insert(T item, IMongoDbTransaction? transaction, CancellationToken cancellationToken = default)
{
item._Etag = BsonObjectIdGenerator.Instance.GenerateId(null, null).ToString()!;
diff --git a/Btms.Backend.IntegrationTests/AnalyticsTests.cs b/Btms.Backend.IntegrationTests/AnalyticsTests.cs
index 1cb9d98..598be38 100644
--- a/Btms.Backend.IntegrationTests/AnalyticsTests.cs
+++ b/Btms.Backend.IntegrationTests/AnalyticsTests.cs
@@ -94,8 +94,6 @@ public async Task GetAllCharts()
var responseDictionary = await response.ToJsonDictionary();
- responseDictionary.Count.Should().Be(13);
-
responseDictionary.Keys.Take(2).Should().Equal(
"importNotificationLinkingByCreated",
"importNotificationLinkingByArrival");
diff --git a/Btms.Backend.Test/Auditing/AuditingTests.cs b/Btms.Backend.Test/Auditing/AuditingTests.cs
index 42bc5c3..8d48281 100644
--- a/Btms.Backend.Test/Auditing/AuditingTests.cs
+++ b/Btms.Backend.Test/Auditing/AuditingTests.cs
@@ -15,7 +15,7 @@ public void CreateAuditWhenDifferentIsDouble()
{
var previous = new TestClassOne { NumberValue = 1.2 };
var current = new TestClassOne { NumberValue = 2.2 };
- var auditEntry = AuditEntry.CreateUpdated(previous, current, "testid", 1, DateTime.UtcNow);
+ var auditEntry = AuditEntry.CreateUpdated(previous, current, "testid", 1, DateTime.UtcNow, AuditEntry.CreatedByIpaffs);
auditEntry.Should().NotBeNull();
auditEntry.Diff.Count.Should().Be(1);
@@ -31,7 +31,7 @@ public void CreateAuditWhenDifferentIsInt()
{
var previous = new TestClassOne { NumberValue = 1 };
var current = new TestClassOne { NumberValue = 2 };
- var auditEntry = AuditEntry.CreateUpdated(previous, current, "testid", 1, DateTime.UtcNow);
+ var auditEntry = AuditEntry.CreateUpdated(previous, current, "testid", 1, DateTime.UtcNow,AuditEntry.CreatedByIpaffs);
auditEntry.Should().NotBeNull();
auditEntry.Diff.Count.Should().Be(1);
diff --git a/Btms.Backend/Config/AnalyticsDashboards.cs b/Btms.Backend/Config/AnalyticsDashboards.cs
index df25f6a..ef4c1c0 100644
--- a/Btms.Backend/Config/AnalyticsDashboards.cs
+++ b/Btms.Backend/Config/AnalyticsDashboards.cs
@@ -69,6 +69,10 @@ public static async Task> GetCharts(
{
"lastMonthImportNotificationsByCommodityCount",
() => importService.ByCommodityCount(DateTime.Today.MonthAgo(), DateTime.Now).AsIDataset()
+ },
+ {
+ "lastMonthDecisionsByStatus",
+ () => movementsService.ByDecision(DateTime.Today.MonthAgo(), DateTime.Now).AsIDataset()
}
};
diff --git a/Btms.Backend/Endpoints/AnalyticsEndpoints.cs b/Btms.Backend/Endpoints/AnalyticsEndpoints.cs
index cfd7cfa..ac42f21 100644
--- a/Btms.Backend/Endpoints/AnalyticsEndpoints.cs
+++ b/Btms.Backend/Endpoints/AnalyticsEndpoints.cs
@@ -16,16 +16,21 @@ public static void UseAnalyticsEndpoints(this IEndpointRouteBuilder app)
{
app.MapGet(BaseRoute + "/dashboard", GetDashboard).AllowAnonymous();
app.MapGet(BaseRoute + "/record-current-state", RecordCurrentState).AllowAnonymous();
- app.MapGet(BaseRoute + "/timeline", Timeline).AllowAnonymous();
+ app.MapGet(BaseRoute + "/timeline", Timeline);
}
private static async Task Timeline(
[FromServices] IImportNotificationsAggregationService importService,
[FromServices] IMovementsAggregationService movementsService,
[FromQuery] string movementId)
{
+ var result = await movementsService.GetHistory(movementId);
+ if (result.HasValue())
+ {
+ return TypedResults.Json(result);
+ }
- return TypedResults.Json(await movementsService.GetHistory(movementId));
+ return Results.NotFound();
}
private static async Task RecordCurrentState(
diff --git a/Btms.Backend/Endpoints/ManagementEndpoints.cs b/Btms.Backend/Endpoints/ManagementEndpoints.cs
index 73d2dd3..61e4f59 100644
--- a/Btms.Backend/Endpoints/ManagementEndpoints.cs
+++ b/Btms.Backend/Endpoints/ManagementEndpoints.cs
@@ -36,7 +36,8 @@ private static bool RedactKeys(string key)
return key.StartsWith("AZURE") ||
key.StartsWith("BlobServiceOptions__Azure") ||
key.StartsWith("ServiceBusOptions__ConnectionString") ||
- key.Contains("password", StringComparison.OrdinalIgnoreCase) ||
+ key.StartsWith("AuthKeyStore__Credentials") ||
+ key.Contains("password", StringComparison.OrdinalIgnoreCase) ||
_keysToRedact.Contains(key);
}
diff --git a/Btms.BlobService/CachingBlobService.cs b/Btms.BlobService/CachingBlobService.cs
index 9019aa6..0a5026b 100644
--- a/Btms.BlobService/CachingBlobService.cs
+++ b/Btms.BlobService/CachingBlobService.cs
@@ -13,7 +13,7 @@ IOptions options
{
public Task CheckBlobAsync(int timeout = default, int retries = default)
{
- return blobService.CheckBlobAsync();
+ return blobService.CheckBlobAsync(timeout, retries);
}
public Task CheckBlobAsync(string uri, int timeout = default, int retries = default)
@@ -23,29 +23,52 @@ public Task CheckBlobAsync(string uri, int timeout = default, int retrie
public async IAsyncEnumerable GetResourcesAsync(string prefix, [EnumeratorCancellation] CancellationToken cancellationToken)
{
- var path = Path.GetFullPath($"{options.Value.CachePath}/{prefix}");
- logger.LogInformation("Scanning disk {Path}", path);
-
- if (Directory.Exists(path))
+ if (!options.Value.CacheReadEnabled)
{
- logger.LogInformation("Folder {Path} exists, looking for files", path);
- foreach (var f in Directory.GetFiles(path, "*.json", SearchOption.AllDirectories))
- {
- var relativePath = Path.GetRelativePath($"{Directory.GetCurrentDirectory()}/{options.Value.CachePath}", f);
- logger.LogInformation("Found file {RelativePath}", relativePath);
- yield return await Task.FromResult(new BtmsBlobItem { Name = relativePath });
- }
+ await foreach (var blobItem in blobService.GetResourcesAsync(prefix, cancellationToken))
+ {
+ yield return blobItem;
+ }
}
- else{
- logger.LogWarning("Cannot scan folder {Path} as it doesn't exist", path);
+ else
+ {
+ var path = Path.GetFullPath($"{options.Value.CachePath}/{prefix}");
+ logger.LogInformation("Scanning disk {Path}", path);
+
+ if (Directory.Exists(path))
+ {
+ logger.LogInformation("Folder {Path} exists, looking for files", path);
+ foreach (var f in Directory.GetFiles(path, "*.json", SearchOption.AllDirectories))
+ {
+ var relativePath = Path.GetRelativePath($"{Directory.GetCurrentDirectory()}/{options.Value.CachePath}", f);
+ logger.LogInformation("Found file {RelativePath}", relativePath);
+ yield return await Task.FromResult(new BtmsBlobItem { Name = relativePath });
+ }
+ }
+ else{
+ logger.LogWarning("Cannot scan folder {Path} as it doesn't exist", path);
+ }
}
}
- public Task GetResource(IBlobItem item, CancellationToken cancellationToken)
+ public async Task GetResource(IBlobItem item, CancellationToken cancellationToken)
{
+ if (!options.Value.CacheReadEnabled)
+ {
+ var blob =blobService.GetResource(item, cancellationToken);
+
+ if (options.Value.CacheWriteEnabled)
+ {
+ item.Content = blob.Result;
+ await CreateBlobAsync(item);
+ }
+
+ return blob.Result;
+ }
+
var filePath = $"{options.Value.CachePath}/{item.Name}";
logger.LogInformation("GetResource {FilePath}", filePath);
- return Task.Run(() => File.ReadAllText(filePath), cancellationToken);
+ return File.ReadAllText(filePath);
}
public async Task CreateBlobsAsync(IBlobItem[] items)
diff --git a/Btms.Business.Tests/Commands/SyncDecisionsCommandTests.cs b/Btms.Business.Tests/Commands/SyncDecisionsCommandTests.cs
index def5cb5..82d9d7a 100644
--- a/Btms.Business.Tests/Commands/SyncDecisionsCommandTests.cs
+++ b/Btms.Business.Tests/Commands/SyncDecisionsCommandTests.cs
@@ -45,7 +45,7 @@ public async Task WhenDecisionBlobsExist_ThenTheyShouldBePlacedOnInternalBus()
await handler.Handle(command, CancellationToken.None);
// ASSERT
- await bus.Received(1).Publish(Arg.Any(), "DECISIONS",
+ await bus.Received(1).Publish(Arg.Any(), "DECISIONS",
Arg.Any>(), Arg.Any());
}
}
diff --git a/Btms.Business.Tests/Services/Decisions/NoMatchDecisionsTest.cs b/Btms.Business.Tests/Services/Decisions/NoMatchDecisionsTest.cs
index 611052f..2931143 100644
--- a/Btms.Business.Tests/Services/Decisions/NoMatchDecisionsTest.cs
+++ b/Btms.Business.Tests/Services/Decisions/NoMatchDecisionsTest.cs
@@ -35,7 +35,8 @@ public async Task WhenClearanceRequest_HasNotMatch_ThenDecisionCodeShouldBeNoMat
decisionResult.Should().NotBeNull();
decisionResult.Decisions.Count.Should().Be(1);
decisionResult.Decisions[0].DecisionCode.Should().Be(DecisionCode.X00);
- await publishBus.Received().Publish(Arg.Any(), Arg.Any(),
+
+ await publishBus.Received().Publish(Arg.Any(), Arg.Any(),
Arg.Any>(), Arg.Any());
await Task.CompletedTask;
}
@@ -48,9 +49,9 @@ private static List GenerateMovements()
var generatorResult = generator.Generate(1, 1, DateTime.UtcNow, config);
- return generatorResult.ClearanceRequests.Select(x =>
+ return generatorResult.Select(x =>
{
- var internalClearanceRequest = AlvsClearanceRequestMapper.Map(x);
+ var internalClearanceRequest = AlvsClearanceRequestMapper.Map((AlvsClearanceRequest)x);
return MovementPreProcessor.BuildMovement(internalClearanceRequest);
}).ToList();
}
diff --git a/Btms.Business.Tests/Services/Matching/MatchingServiceTests.cs b/Btms.Business.Tests/Services/Matching/MatchingServiceTests.cs
index 9a6f19c..b55825c 100644
--- a/Btms.Business.Tests/Services/Matching/MatchingServiceTests.cs
+++ b/Btms.Business.Tests/Services/Matching/MatchingServiceTests.cs
@@ -2,6 +2,7 @@
using Btms.Business.Services.Matching;
using Btms.Model;
using Btms.Model.Ipaffs;
+using Btms.Types.Alvs;
using Btms.Types.Alvs.Mapping;
using Btms.Types.Ipaffs.Mapping;
using FluentAssertions;
@@ -52,9 +53,9 @@ private static List GenerateMovements()
var generatorResult = generator.Generate(1, 1, DateTime.UtcNow, config);
- return generatorResult.ClearanceRequests.Select(x =>
+ return generatorResult.Select(x =>
{
- var internalClearanceRequest = AlvsClearanceRequestMapper.Map(x);
+ var internalClearanceRequest = AlvsClearanceRequestMapper.Map((AlvsClearanceRequest)x);
return MovementPreProcessor.BuildMovement(internalClearanceRequest);
}).ToList();
}
@@ -67,15 +68,31 @@ private static (List Notifications, List Movements
var generatorResult = generator.Generate(1, 1, DateTime.UtcNow, config);
- var movements = generatorResult.ClearanceRequests.Select(x =>
- {
- var internalClearanceRequest = AlvsClearanceRequestMapper.Map(x);
- return MovementPreProcessor.BuildMovement(internalClearanceRequest);
- }).ToList();
-
- var notifications = generatorResult.ImportNotifications.Select(ImportNotificationMapper.Map).ToList();
-
- return new ValueTuple, List>(notifications, movements);
+ var messages = generatorResult.Aggregate(
+ new { Notifications = new List(), Movements = new List(), }, (memo, x) =>
+ {
+ switch (x)
+ {
+ case null:
+ throw new ArgumentNullException();
+
+ case Btms.Types.Ipaffs.ImportNotification n:
+ var internalNotification = ImportNotificationMapper.Map(n);
+ memo.Notifications.Add(internalNotification);
+ break;
+ case AlvsClearanceRequest cr:
+
+ var internalClearanceRequest = AlvsClearanceRequestMapper.Map(cr);
+ memo.Movements.Add(MovementPreProcessor.BuildMovement(internalClearanceRequest));
+ break;
+ default:
+ throw new ArgumentException($"Unexpected type {x.GetType().Name}");
+ }
+
+ return memo;
+ });
+
+ return new ValueTuple, List>(messages.Notifications, messages.Movements);
}
diff --git a/Btms.Business/Commands/SyncDecisionsCommand.cs b/Btms.Business/Commands/SyncDecisionsCommand.cs
index c9a5497..ada3ff8 100644
--- a/Btms.Business/Commands/SyncDecisionsCommand.cs
+++ b/Btms.Business/Commands/SyncDecisionsCommand.cs
@@ -26,7 +26,7 @@ public override async Task Handle(SyncDecisionsCommand request, CancellationToke
var rootFolder = string.IsNullOrEmpty(request.RootFolder)
? Options.DmpBlobRootFolder
: request.RootFolder;
- await SyncBlobPaths(request.SyncPeriod, "DECISIONS", request.JobId,
+ await SyncBlobPaths(request.SyncPeriod, "DECISIONS", request.JobId,
cancellationToken,$"{rootFolder}/DECISIONS");
}
}
diff --git a/Btms.Business/Pipelines/PreProcessing/MovementPreProcessor.cs b/Btms.Business/Pipelines/PreProcessing/MovementPreProcessor.cs
index 85c3609..9f39852 100644
--- a/Btms.Business/Pipelines/PreProcessing/MovementPreProcessor.cs
+++ b/Btms.Business/Pipelines/PreProcessing/MovementPreProcessor.cs
@@ -24,7 +24,8 @@ public async Task> Process(PreProcessingContext> Process(PreProcessingContext
diff --git a/Btms.Business/Services/Decisions/DecisionMessageBuilder.cs b/Btms.Business/Services/Decisions/DecisionMessageBuilder.cs
index 49f1751..e91d4e6 100644
--- a/Btms.Business/Services/Decisions/DecisionMessageBuilder.cs
+++ b/Btms.Business/Services/Decisions/DecisionMessageBuilder.cs
@@ -1,14 +1,15 @@
using Btms.Model;
using Btms.Model.Ipaffs;
using Btms.Types.Alvs;
+using Decision = Btms.Types.Alvs.Decision;
namespace Btms.Business.Services.Decisions;
public static class DecisionMessageBuilder
{
- public static Task> Build(DecisionContext decisionContext, DecisionResult decisionResult)
+ public static Task> Build(DecisionContext decisionContext, DecisionResult decisionResult)
{
- var list = new List();
+ var list = new List();
var movements = decisionResult.Decisions.GroupBy(x => x.MovementId).ToList();
@@ -16,7 +17,7 @@ public static Task> Build(DecisionContext decisionCon
{
var movement = decisionContext.Movements.First(x => x.Id == movementGroup.Key);
var messageNumber = movement is { Decisions: null } ? 1 : movement.Decisions.Count + 1;
- var decisionMessage = new AlvsClearanceRequest()
+ var decisionMessage = new Decision()
{
ServiceHeader = BuildServiceHeader(),
Header = BuildHeader(movement, messageNumber),
@@ -48,10 +49,7 @@ private static Header BuildHeader(Movement movement, int messageNumber)
DecisionNumber = messageNumber
};
}
-
-
-
-
+
private static IEnumerable BuildItems(Movement movement, IGrouping movementGroup)
{
var itemGroups = movementGroup.GroupBy(x => x.ItemNumber);
diff --git a/Btms.Common/Extensions/DateTimeExtensions.cs b/Btms.Common/Extensions/DateTimeExtensions.cs
index 1a453f3..9c9623a 100644
--- a/Btms.Common/Extensions/DateTimeExtensions.cs
+++ b/Btms.Common/Extensions/DateTimeExtensions.cs
@@ -69,9 +69,10 @@ private static int CreateRandomInt(int min, int max)
return Random.Shared.Next(min, max);
}
- public static DateTime RandomTime(this DateTime dt)
+ public static DateTime RandomTime(this DateTime dt, int maxHour = 23)
{
- return new DateTime(dt.Year, dt.Month, dt.Day, CreateRandomInt(0,23), CreateRandomInt(0, 60), CreateRandomInt(0, 60), dt.Kind);
+ return new DateTime(dt.Year, dt.Month, dt.Day,
+ CreateRandomInt(0, maxHour), CreateRandomInt(0, 60), CreateRandomInt(0, 60), dt.Kind);
}
public static DateOnly ToDate(this DateTime val)
diff --git a/Btms.Consumers.Tests/ClearanceRequestConsumerTests.cs b/Btms.Consumers.Tests/ClearanceRequestConsumerTests.cs
index da8edc4..f7219ef 100644
--- a/Btms.Consumers.Tests/ClearanceRequestConsumerTests.cs
+++ b/Btms.Consumers.Tests/ClearanceRequestConsumerTests.cs
@@ -30,7 +30,7 @@ public async Task WhenPreProcessingSucceeds_AndLastAuditEntryIsLinked_ThenLinkSh
var movement =
MovementPreProcessor.BuildMovement(AlvsClearanceRequestMapper.Map(clearanceRequest));
- movement.Update(AuditEntry.CreateLinked("Test", 1, DateTime.Now));
+ movement.Update(AuditEntry.CreateLinked("Test", 1));
var mockLinkingService = Substitute.For();
var decisionService = Substitute.For();
@@ -67,7 +67,7 @@ public async Task WhenPreProcessingSucceeds_AndLastAuditEntryIsCreated_ThenLinkS
var movement =
MovementPreProcessor.BuildMovement(AlvsClearanceRequestMapper.Map(clearanceRequest));
- movement.Update(AuditEntry.CreateCreatedEntry(movement,"Test", 1, DateTime.Now));
+ movement.Update(AuditEntry.CreateCreatedEntry(movement,"Test", 1, DateTime.Now, AuditEntry.CreatedByCds));
var mockLinkingService = Substitute.For();
var decisionService = Substitute.For();
diff --git a/Btms.Consumers.Tests/NotificationsConsumerTests.cs b/Btms.Consumers.Tests/NotificationsConsumerTests.cs
index 067e95d..9d6b11b 100644
--- a/Btms.Consumers.Tests/NotificationsConsumerTests.cs
+++ b/Btms.Consumers.Tests/NotificationsConsumerTests.cs
@@ -27,7 +27,7 @@ public async Task WhenPreProcessingSucceeds_AndLastAuditEntryIsLinked_ThenLinkSh
// ARRANGE
var notification = CreateImportNotification();
var modelNotification = notification.MapWithTransform();
- modelNotification.Changed(AuditEntry.CreateLinked("Test", 1, DateTime.Now));
+ modelNotification.Changed(AuditEntry.CreateLinked("Test", 1));
var mockLinkingService = Substitute.For();
var decisionService = Substitute.For();
var matchingService = Substitute.For();
@@ -57,7 +57,7 @@ public async Task WhenPreProcessingSucceeds_AndLastAuditEntryIsCreated_ThenLinkS
// ARRANGE
var notification = CreateImportNotification();
var modelNotification = notification.MapWithTransform();
- modelNotification.Changed(AuditEntry.CreateCreatedEntry(modelNotification, "Test", 1, DateTime.Now));
+ modelNotification.Changed(AuditEntry.CreateCreatedEntry(modelNotification, "Test", 1, DateTime.Now, AuditEntry.CreatedByIpaffs));
var mockLinkingService = Substitute.For();
var decisionService = Substitute.For();
var matchingService = Substitute.For();
diff --git a/Btms.Consumers/AlvsClearanceRequestConsumer.cs b/Btms.Consumers/AlvsClearanceRequestConsumer.cs
index 404ea7b..f209a39 100644
--- a/Btms.Consumers/AlvsClearanceRequestConsumer.cs
+++ b/Btms.Consumers/AlvsClearanceRequestConsumer.cs
@@ -42,10 +42,10 @@ public async Task OnHandle(AlvsClearanceRequest message)
Context.Linked();
}
- var matchResult = await matchingService.Process(
- new MatchingContext(linkResult.Notifications, linkResult.Movements), Context.CancellationToken);
+ var matchResult = await matchingService.Process(
+ new MatchingContext(linkResult.Notifications, linkResult.Movements), Context.CancellationToken);
- await decisionService.Process(new DecisionContext(linkResult.Notifications, linkResult.Movements, matchResult, true), Context.CancellationToken);
+ await decisionService.Process(new DecisionContext(linkResult.Notifications, linkResult.Movements, matchResult, true), Context.CancellationToken);
}
}
}
diff --git a/Btms.Consumers/ConsumerOptions.cs b/Btms.Consumers/ConsumerOptions.cs
index 7b64d5a..f9fdde9 100644
--- a/Btms.Consumers/ConsumerOptions.cs
+++ b/Btms.Consumers/ConsumerOptions.cs
@@ -4,6 +4,7 @@ public class ConsumerOptions
{
public const string SectionName = nameof(ConsumerOptions);
+ public bool EnableBlockingPublish { get; set; } = false;
public int InMemoryNotifications { get; set; } = 2;
public int InMemoryGmrs { get; set; } = 2;
public int InMemoryClearanceRequests { get; set; } = 2;
diff --git a/Btms.Consumers/DecisionsConsumer.cs b/Btms.Consumers/DecisionsConsumer.cs
index b1fe7e0..8ce457c 100644
--- a/Btms.Consumers/DecisionsConsumer.cs
+++ b/Btms.Consumers/DecisionsConsumer.cs
@@ -6,11 +6,11 @@
namespace Btms.Consumers;
public class DecisionsConsumer(IMongoDbContext dbContext)
- : IConsumer, IConsumerWithContext
+ : IConsumer, IConsumerWithContext
{
- public async Task OnHandle(AlvsClearanceRequest message)
+ public async Task OnHandle(Decision message)
{
- var internalClearanceRequest = AlvsClearanceRequestMapper.Map(message);
+ var internalClearanceRequest = DecisionMapper.Map(message);
var existingMovement = await dbContext.Movements.Find(message.Header!.EntryReference!);
if (existingMovement != null)
diff --git a/Btms.Consumers/Extensions/ServiceCollectionExtensions.cs b/Btms.Consumers/Extensions/ServiceCollectionExtensions.cs
index 799d2c3..b643580 100644
--- a/Btms.Consumers/Extensions/ServiceCollectionExtensions.cs
+++ b/Btms.Consumers/Extensions/ServiceCollectionExtensions.cs
@@ -10,6 +10,7 @@
using SlimMessageBus.Host.Interceptor;
using SlimMessageBus.Host.Memory;
using AlvsClearanceRequest = Btms.Types.Alvs.AlvsClearanceRequest;
+using Decision = Btms.Types.Alvs.Decision;
namespace Btms.Consumers.Extensions
{
@@ -41,7 +42,7 @@ public static IServiceCollection AddConsumers(this IServiceCollection services,
{
cbb.WithProviderMemory(cfg =>
{
- cfg.EnableBlockingPublish = false;
+ cfg.EnableBlockingPublish = consumerOpts.EnableBlockingPublish;
cfg.EnableMessageHeaders = true;
})
.AddServicesFromAssemblyContaining(
@@ -64,7 +65,8 @@ public static IServiceCollection AddConsumers(this IServiceCollection services,
x.Instances(consumerOpts.InMemoryClearanceRequests);
x.Topic("CLEARANCEREQUESTS").WithConsumer();
})
- .Consume(x =>
+ .Produce(x => x.DefaultTopic(nameof(Decision)))
+ .Consume(x =>
{
x.Instances(consumerOpts.InMemoryDecisions);
x.Topic("DECISIONS").WithConsumer();
diff --git a/Btms.Consumers/GmrConsumer.cs b/Btms.Consumers/GmrConsumer.cs
index 98832dc..be078f4 100644
--- a/Btms.Consumers/GmrConsumer.cs
+++ b/Btms.Consumers/GmrConsumer.cs
@@ -19,7 +19,7 @@ public async Task OnHandle(SearchGmrsForDeclarationIdsResponse message)
if (existingGmr is null)
{
var auditEntry =
- AuditEntry.CreateCreatedEntry(internalGmr, auditId!, 1, gmr.UpdatedSource);
+ AuditEntry.CreateCreatedEntry(internalGmr, auditId!, 1, gmr.UpdatedSource, AuditEntry.CreatedByGvms);
internalGmr.AuditEntries.Add(auditEntry);
await dbContext.Gmrs.Insert(internalGmr);
}
@@ -33,7 +33,8 @@ public async Task OnHandle(SearchGmrsForDeclarationIdsResponse message)
current: internalGmr,
id: auditId!,
version: internalGmr.AuditEntries.Count + 1,
- lastUpdated: gmr.UpdatedSource);
+ lastUpdated: gmr.UpdatedSource,
+ AuditEntry.CreatedByGvms);
internalGmr.AuditEntries.Add(auditEntry);
await dbContext.Gmrs.Update(internalGmr, existingGmr._Etag);
}
diff --git a/Btms.Consumers/NotificationConsumer.cs b/Btms.Consumers/NotificationConsumer.cs
index 13fc6c5..8a17ab8 100644
--- a/Btms.Consumers/NotificationConsumer.cs
+++ b/Btms.Consumers/NotificationConsumer.cs
@@ -42,10 +42,10 @@ public async Task OnHandle(ImportNotification message)
Context.Linked();
}
- var matchResult = await matchingService.Process(
- new MatchingContext(linkResult.Notifications, linkResult.Movements), Context.CancellationToken);
+ var matchResult = await matchingService.Process(
+ new MatchingContext(linkResult.Notifications, linkResult.Movements), Context.CancellationToken);
- await decisionService.Process(new DecisionContext(linkResult.Notifications, linkResult.Movements, matchResult), Context.CancellationToken);
+ await decisionService.Process(new DecisionContext(linkResult.Notifications, linkResult.Movements, matchResult), Context.CancellationToken);
}
}
diff --git a/Btms.Model/Auditing/AuditEntry.cs b/Btms.Model/Auditing/AuditEntry.cs
index 2c56320..7eb9f92 100644
--- a/Btms.Model/Auditing/AuditEntry.cs
+++ b/Btms.Model/Auditing/AuditEntry.cs
@@ -8,9 +8,13 @@ namespace Btms.Model.Auditing;
public class AuditEntry
{
- private const string CreatedBySystem = "System";
+ public const string CreatedBySystem = "Btms";
+ public const string CreatedByIpaffs = "Ipaffs";
+ public const string CreatedByAlvs = "Alvs";
+ public const string CreatedByCds = "Cds";
+ public const string CreatedByGvms = "Gvms";
public string Id { get; set; } = default!;
- public int Version { get; set; }
+ public int? Version { get; set; }
public string CreatedBy { get; set; } = default!;
@@ -21,6 +25,8 @@ public class AuditEntry
public string Status { get; set; } = default!;
public List Diff { get; set; } = new();
+
+ public Dictionary> Context { get; set; } = new();
public bool IsCreatedOrUpdated()
{
@@ -39,27 +45,27 @@ public bool IsUpdated()
public static AuditEntry Create(T previous, T current, string id, int version, DateTime? lastUpdated,
- string lastUpdatedBy, string status)
+ string lastUpdatedBy, string status, string source)
{
var node1 = JsonNode.Parse(previous.ToJsonString());
var node2 = JsonNode.Parse(current.ToJsonString());
- return CreateInternal(node1!, node2!, id, version, lastUpdated, status);
+ return CreateInternal(node1!, node2!, id, version, lastUpdated, status, source);
}
- public static AuditEntry CreateUpdated(T previous, T current, string id, int version, DateTime? lastUpdated)
+ public static AuditEntry CreateUpdated(T previous, T current, string id, int version, DateTime? lastUpdated, string source)
{
- return Create(previous, current, id, version, lastUpdated, CreatedBySystem, "Updated");
+ return Create(previous, current, id, version, lastUpdated, CreatedBySystem, "Updated", source);
}
- public static AuditEntry CreateUpdated(ChangeSet changeSet, string id, int version, DateTime? lastUpdated)
+ public static AuditEntry CreateUpdated(ChangeSet changeSet, string id, int version, DateTime? lastUpdated, string source)
{
var auditEntry = new AuditEntry
{
Id = id,
Version = version,
CreatedSource = lastUpdated,
- CreatedBy = CreatedBySystem,
+ CreatedBy = source,
CreatedLocal = DateTime.UtcNow,
Status = "Updated"
};
@@ -72,74 +78,75 @@ public static AuditEntry CreateUpdated(ChangeSet changeSet, string id, int versi
return auditEntry;
}
- public static AuditEntry CreateCreatedEntry(T current, string id, int version, DateTime? lastUpdated)
+ public static AuditEntry CreateCreatedEntry(T current, string id, int version, DateTime? lastUpdated, string source)
{
return new AuditEntry
{
Id = id,
Version = version,
CreatedSource = lastUpdated,
- CreatedBy = CreatedBySystem,
+ CreatedBy = source,
CreatedLocal = DateTime.UtcNow,
Status = "Created"
};
}
- public static AuditEntry CreateSkippedVersion(string id, int version, DateTime? lastUpdated)
+ public static AuditEntry CreateSkippedVersion(string id, int version, DateTime? lastUpdated, string source)
{
return new AuditEntry
{
Id = id,
Version = version,
CreatedSource = lastUpdated,
- CreatedBy = CreatedBySystem,
+ CreatedBy = source,
CreatedLocal = DateTime.UtcNow,
Status = "Updated"
};
}
- public static AuditEntry CreateLinked(string id, int version, DateTime? lastUpdated)
+ public static AuditEntry CreateLinked(string id, int version)
{
+ var t = DateTime.UtcNow;
return new AuditEntry
{
Id = id,
- Version = version,
- CreatedSource = lastUpdated,
+ CreatedSource = t,
CreatedBy = CreatedBySystem,
- CreatedLocal = DateTime.UtcNow,
+ CreatedLocal = t,
Status = "Linked"
};
}
- public static AuditEntry CreateMatch(string id, int version, DateTime? lastUpdated)
+ public static AuditEntry CreateMatch(string id, int version)
{
+ var t = DateTime.UtcNow;
return new AuditEntry
{
Id = id,
- Version = version,
- CreatedSource = lastUpdated,
+ CreatedSource = t,
CreatedBy = CreatedBySystem,
- CreatedLocal = DateTime.UtcNow,
+ CreatedLocal = t,
Status = "Matched"
};
}
public static AuditEntry CreateDecision(string id, int version,
- DateTime? lastUpdated, string lastUpdatedBy)
+ DateTime? lastUpdated, string lastUpdatedBy, Dictionary> context, bool isAlvs)
{
return new AuditEntry()
{
Id = id,
- Version = version,
CreatedSource = lastUpdated,
- CreatedBy = CreatedBySystem,
+ CreatedBy = isAlvs ? CreatedByAlvs : CreatedBySystem,
CreatedLocal = DateTime.UtcNow,
- Status = "Decision"
+ Status = "Decision",
+ Context = context
+
};
}
private static AuditEntry CreateInternal(JsonNode previous, JsonNode current, string id, int version,
- DateTime? lastUpdated, string status)
+ DateTime? lastUpdated, string status, string source)
{
var diff = previous.CreatePatch(current);
@@ -148,7 +155,7 @@ private static AuditEntry CreateInternal(JsonNode previous, JsonNode current, st
Id = id,
Version = version,
CreatedSource = lastUpdated,
- CreatedBy = CreatedBySystem,
+ CreatedBy = source,
CreatedLocal = DateTime.UtcNow,
Status = status
};
diff --git a/Btms.Model/Ipaffs/ImportNotification.cs b/Btms.Model/Ipaffs/ImportNotification.cs
index 3c94ae1..c9ac2a0 100644
--- a/Btms.Model/Ipaffs/ImportNotification.cs
+++ b/Btms.Model/Ipaffs/ImportNotification.cs
@@ -144,7 +144,7 @@ public void AddRelationship(TdmRelationshipObject relationship)
if (linked)
{
- AuditEntries.Add(AuditEntry.CreateLinked(string.Empty, Version.GetValueOrDefault(), UpdatedSource));
+ AuditEntries.Add(AuditEntry.CreateLinked(string.Empty, Version.GetValueOrDefault()));
}
}
@@ -159,7 +159,8 @@ public void Create(string auditId)
this,
auditId,
Version.GetValueOrDefault(),
- UpdatedSource);
+ UpdatedSource,
+ AuditEntry.CreatedByIpaffs);
Changed(auditEntry);
}
@@ -168,7 +169,8 @@ public void Skipped(string auditId, int version)
var auditEntry = AuditEntry.CreateSkippedVersion(
auditId,
version,
- UpdatedSource);
+ UpdatedSource,
+ AuditEntry.CreatedByIpaffs);
Changed(auditEntry);
}
@@ -177,7 +179,8 @@ public void Update(string auditId, ChangeSet changeSet)
var auditEntry = AuditEntry.CreateUpdated(changeSet,
auditId,
Version.GetValueOrDefault(),
- UpdatedSource);
+ UpdatedSource,
+ AuditEntry.CreatedByIpaffs);
Changed(auditEntry);
}
diff --git a/Btms.Model/Movement.cs b/Btms.Model/Movement.cs
index 775827a..0b70088 100644
--- a/Btms.Model/Movement.cs
+++ b/Btms.Model/Movement.cs
@@ -111,7 +111,7 @@ public void AddRelationship(TdmRelationshipObject relationship)
if (linked)
{
- AuditEntries.Add(AuditEntry.CreateLinked(String.Empty, this.AuditEntries.FirstOrDefault()?.Version ?? 1, UpdatedSource));
+ AuditEntries.Add(AuditEntry.CreateLinked(String.Empty, this.AuditEntries.FirstOrDefault()?.Version ?? 1));
}
}
@@ -136,11 +136,23 @@ public bool MergeDecision(string path, AlvsClearanceRequest clearanceRequest)
}
}
+ var decisionAuditContext = new Dictionary>();
+ decisionAuditContext.Add("movements", new Dictionary()
+ {
+ { clearanceRequest.Header!.EntryReference!, clearanceRequest.Header!.EntryVersionNumber!.ToString()! }
+ });
+ decisionAuditContext.Add("importNotifications", new Dictionary()
+ {
+ { "todo", "todo" }
+ });
+
var auditEntry = AuditEntry.CreateDecision(
BuildNormalizedDecisionPath(path),
clearanceRequest.Header!.EntryVersionNumber.GetValueOrDefault(),
clearanceRequest.ServiceHeader!.ServiceCalled,
- clearanceRequest.Header.DeclarantName!);
+ clearanceRequest.Header.DeclarantName!,
+ decisionAuditContext,
+ clearanceRequest.ServiceHeader?.SourceSystem != "BTMS");
Decisions ??= [];
Decisions.Add(clearanceRequest);
diff --git a/Btms.SyncJob/Extensions/ServiceProviderExtensions.cs b/Btms.SyncJob/Extensions/ServiceProviderExtensions.cs
new file mode 100644
index 0000000..2829093
--- /dev/null
+++ b/Btms.SyncJob/Extensions/ServiceProviderExtensions.cs
@@ -0,0 +1,24 @@
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging;
+
+namespace Btms.SyncJob.Extensions;
+
+public static class ServiceProviderExtensions
+{
+ public static async Task WaitOnAllJobs(this IServiceProvider provider, ILogger logger)
+ {
+ var store = provider.GetService()!;
+
+ var complete = false;
+ while (!complete)
+ {
+ var jobs = store.GetJobs();
+ // logger.LogInformation(jobs.ToJsonString());
+ logger.LogInformation("{jobs} found.", jobs.Count);
+
+ complete = true;
+ }
+
+ return await Task.FromResult(true);
+ }
+}
\ No newline at end of file
diff --git a/Btms.Tests.Common/Btms.Tests.Common.csproj b/Btms.Tests.Common/Btms.Tests.Common.csproj
new file mode 100644
index 0000000..756957c
--- /dev/null
+++ b/Btms.Tests.Common/Btms.Tests.Common.csproj
@@ -0,0 +1,18 @@
+
+
+
+ net8.0
+ enable
+ enable
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/Btms.Tests.Common/Class1.cs b/Btms.Tests.Common/Class1.cs
new file mode 100644
index 0000000..70636b9
--- /dev/null
+++ b/Btms.Tests.Common/Class1.cs
@@ -0,0 +1,5 @@
+namespace Btms.Tests.Common;
+
+public class Class1
+{
+}
diff --git a/Btms.Types.Alvs.Mapping.V1/DecisionMapper.g.cs b/Btms.Types.Alvs.Mapping.V1/DecisionMapper.g.cs
new file mode 100644
index 0000000..4216d01
--- /dev/null
+++ b/Btms.Types.Alvs.Mapping.V1/DecisionMapper.g.cs
@@ -0,0 +1,32 @@
+//------------------------------------------------------------------------------
+//
+ // This code was generated from a template.
+ //
+ // Manual changes to this file may cause unexpected behavior in your application.
+ // Manual changes to this file will be overwritten if the code is regenerated.
+ //
+//
+//------------------------------------------------------------------------------
+#nullable enable
+
+
+namespace Btms.Types.Alvs.Mapping;
+
+public static class DecisionMapper
+{
+ public static Btms.Model.Alvs.AlvsClearanceRequest Map(Btms.Types.Alvs.Decision from)
+ {
+ if (from is null)
+ {
+ return default!;
+ }
+
+ var to = new Btms.Model.Alvs.AlvsClearanceRequest();
+ to.ServiceHeader = ServiceHeaderMapper.Map(from?.ServiceHeader!);
+ to.Header = HeaderMapper.Map(from?.Header!);
+ to.Items = from?.Items?.Select(x => ItemsMapper.Map(x)).ToArray();
+ return to;
+ }
+}
+
+
diff --git a/Btms.Types.Alvs.V1/ClearanceRequestExtensions.cs b/Btms.Types.Alvs.V1/ClearanceRequestExtensions.cs
index 0c85cc8..e31dd49 100644
--- a/Btms.Types.Alvs.V1/ClearanceRequestExtensions.cs
+++ b/Btms.Types.Alvs.V1/ClearanceRequestExtensions.cs
@@ -44,6 +44,6 @@ public override DateTime Read(ref Utf8JsonReader reader, Type typeToConvert, Jso
public override void Write(Utf8JsonWriter writer, DateTime value, JsonSerializerOptions options)
{
- writer.WriteStringValue(new DateTimeOffset(value).ToUnixTimeMilliseconds().ToString());
+ writer.WriteNumberValue(new DateTimeOffset(value).ToUnixTimeMilliseconds());
}
}
\ No newline at end of file
diff --git a/Btms.Types.Alvs.V1/Decision.cs b/Btms.Types.Alvs.V1/Decision.cs
new file mode 100644
index 0000000..3ab8861
--- /dev/null
+++ b/Btms.Types.Alvs.V1/Decision.cs
@@ -0,0 +1,42 @@
+
+#nullable enable
+
+using System.Text.Json.Serialization;
+using System.Dynamic;
+
+
+namespace Btms.Types.Alvs;
+
+///
+/// This is a copy of the AlvsClearanceRequest
+/// As a temporary measure to allow us to distinguish between the two types when
+/// selecting consumers via the DI container. We'll also start to
+/// Plan is to follow the same approach as other types (currently generated) in time.
+///
+public class Decision //
+{
+
+
+ ///
+ ///
+ ///
+ [JsonPropertyName("serviceHeader")]
+ public ServiceHeader? ServiceHeader { get; set; }
+
+
+ ///
+ ///
+ ///
+ [JsonPropertyName("header")]
+ public Header? Header { get; set; }
+
+
+ ///
+ ///
+ ///
+ [JsonPropertyName("items")]
+ public Items[]? Items { get; set; }
+
+ }
+
+
diff --git a/TestDataGenerator.Tests/ClearanceRequestBuilderTests.cs b/TestDataGenerator.Tests/ClearanceRequestBuilderTests.cs
index 6e81162..24ff92a 100644
--- a/TestDataGenerator.Tests/ClearanceRequestBuilderTests.cs
+++ b/TestDataGenerator.Tests/ClearanceRequestBuilderTests.cs
@@ -20,7 +20,7 @@ public void WithEntryDate_ShouldSet()
{
var date = DateTime.Today.AddDays(-5);
var builder = ClearanceRequestBuilder.Default();
- builder.WithEntryDate(date);
+ builder.WithCreationDate(date);
var cr = builder.Build();
cr.ServiceHeader!.ServiceCallTimestamp.ToDate().Should().Be(date.ToDate());
diff --git a/TestDataGenerator.Tests/ImportNotificationBuilderTests.cs b/TestDataGenerator.Tests/ImportNotificationBuilderTests.cs
index d39de6e..b98e182 100644
--- a/TestDataGenerator.Tests/ImportNotificationBuilderTests.cs
+++ b/TestDataGenerator.Tests/ImportNotificationBuilderTests.cs
@@ -39,7 +39,7 @@ public void WithEntryDate_ShouldSet()
{
var date = DateTime.Today.AddDays(-5);
var builder = ImportNotificationBuilder.Default();
- builder.WithEntryDate(date);
+ builder.WithCreationDate(date);
var notification = builder.Build();
notification.LastUpdated.ToDate().Should().Be(date.ToDate());
diff --git a/TestDataGenerator/ClearanceRequestBuilder.cs b/TestDataGenerator/ClearanceRequestBuilder.cs
index 0f0d441..34294a0 100644
--- a/TestDataGenerator/ClearanceRequestBuilder.cs
+++ b/TestDataGenerator/ClearanceRequestBuilder.cs
@@ -44,11 +44,19 @@ public ClearanceRequestBuilder WithReferenceNumber(string chedReference)
});
}
- public ClearanceRequestBuilder WithEntryDate(DateTime entryDate)
+ public ClearanceRequestBuilder WithCreationDate(DateTime entryDate, bool randomTime = true)
{
- return Do(x => x.ServiceHeader!.ServiceCallTimestamp = entryDate.RandomTime());
+ var entry = randomTime ?
+ // We don't want documents created in the future!
+ entryDate.RandomTime(entryDate.Date == DateTime.Today ? DateTime.Now.AddHours(-2).Hour : 23)
+ : entryDate;
+
+ return Do(x => x.ServiceHeader!.ServiceCallTimestamp = entry);
+ }
+ public ClearanceRequestBuilder WithEntryVersionNumber(int version)
+ {
+ return Do(x => x.Header!.EntryVersionNumber = version);
}
-
public ClearanceRequestBuilder WithArrivalDateTimeOffset(DateOnly? date, TimeOnly? time,
int maxHoursOffset = 12, int maxMinsOffset = 30)
{
@@ -65,7 +73,7 @@ public ClearanceRequestBuilder WithArrivalDateTimeOffset(DateOnly? date, Time
}
public ClearanceRequestBuilder WithItem(string documentCode, string commodityCode, string description,
- int netWeight)
+ int netWeight, string checkCode = "H2019")
{
return Do(cr =>
{
@@ -73,6 +81,7 @@ public ClearanceRequestBuilder WithItem(string documentCode, string commodity
cr.Items![0].GoodsDescription = description;
cr.Items![0].ItemNetMass = netWeight;
cr.Items![0].Documents![0].DocumentCode = documentCode;
+ cr.Items![0].Checks![0].CheckCode = checkCode;
});
}
diff --git a/TestDataGenerator/DecisionBuilder.cs b/TestDataGenerator/DecisionBuilder.cs
new file mode 100644
index 0000000..16a7eaa
--- /dev/null
+++ b/TestDataGenerator/DecisionBuilder.cs
@@ -0,0 +1,87 @@
+using Btms.Common.Extensions;
+using Btms.Model;
+using Btms.Types.Alvs;
+using TestDataGenerator.Helpers;
+
+namespace TestDataGenerator;
+
+public class DecisionBuilder(string file) : DecisionBuilder(file);
+
+public class DecisionBuilder : BuilderBase>
+ where T : Decision, new()
+{
+ private DecisionBuilder()
+ {
+ }
+
+ protected DecisionBuilder(string file) : base(file)
+ {
+ }
+
+ public static DecisionBuilder Default()
+ {
+ return new DecisionBuilder();
+ }
+
+ public static DecisionBuilder FromFile(string file)
+ {
+ return new DecisionBuilder(file);
+ }
+
+ public DecisionBuilder WithReferenceNumber(string chedReference)
+ {
+ var id = MatchIdentifier.FromNotification(chedReference);
+ return Do(x =>
+ {
+ x.Header!.EntryReference = id.AsCdsEntryReference();
+ x.Header!.DeclarationUcr = id.AsCdsDeclarationUcr();
+ x.Header!.MasterUcr = id.AsCdsMasterUcr();
+ });
+ }
+
+ public DecisionBuilder WithDecisionNumber(int number)
+ {
+ return Do(x => x.Header!.DecisionNumber = number);
+ }
+
+ public DecisionBuilder WithCreationDate(DateTime entryDate, bool randomTime = true)
+ {
+ var entry = randomTime ?
+ // We don't want documents created in the future!
+ entryDate.RandomTime(entryDate.Date == DateTime.Today ? DateTime.Now.AddHours(-2).Hour : 23)
+ : entryDate;
+
+ return Do(x => x.ServiceHeader!.ServiceCallTimestamp = entry);
+ }
+ public DecisionBuilder WithEntryVersionNumber(int version)
+ {
+ return Do(x => x.Header!.EntryVersionNumber = version);
+ }
+
+ public DecisionBuilder WithItemAndCheck(int item, string checkCode, string decisionCode)
+ {
+ return Do(dec =>
+ {
+ dec.Items![0].ItemNumber = item;
+ dec.Items![0].Checks![0].CheckCode = checkCode;
+ dec.Items![0].Checks![0].DecisionCode = decisionCode;
+ });
+ }
+
+ protected override DecisionBuilder Validate()
+ {
+ return Do(cr =>
+ {
+ cr.ServiceHeader!.ServiceCallTimestamp.AssertHasValue("Decision ServiceCallTimestamp missing");
+ cr.Header!.EntryReference.AssertHasValue("Decision EntryReference missing");
+ cr.Header!.DecisionNumber.AssertHasValue("Decision DecisionNumber missing");
+
+ Array.ForEach(cr.Items!, i =>
+ Array.ForEach(i.Checks!, c =>
+ {
+ c.CheckCode.AssertHasValue();
+ c.DecisionCode.AssertHasValue();
+ }));
+ });
+ }
+}
\ No newline at end of file
diff --git a/TestDataGenerator/Generator.cs b/TestDataGenerator/Generator.cs
index 319c35c..91df6fe 100644
--- a/TestDataGenerator/Generator.cs
+++ b/TestDataGenerator/Generator.cs
@@ -41,17 +41,18 @@ public async Task Generate(int scenario, ScenarioConfig config, string rootPath)
private async Task InsertToBlobStorage(ScenarioGenerator.GeneratorResult result, string rootPath)
{
logger.LogInformation(
- "Uploading {ImportNotificationsLength} Notification(s) and {ClearanceRequestsLength} Clearance Request(s) to blob storage",
- result.ImportNotifications.Length, result.ClearanceRequests.Length);
-
- var importNotificationBlobItems = result.ImportNotifications.Select(n =>
- new BtmsBlobItem { Name = n.BlobPath(rootPath), Content = JsonSerializer.Serialize(n) });
-
- var alvsClearanceRequestBlobItems = result.ClearanceRequests.Select(cr =>
- new BtmsBlobItem { Name = cr.BlobPath(rootPath), Content = JsonSerializer.Serialize(cr) });
+ "Uploading {Count} messages to blob storage",
+ result.Count);
+
+ var blobs = result.Select(r => new BtmsBlobItem
+ {
+ Name = r.BlobPath(rootPath), Content = JsonSerializer.Serialize(r)
+ });
- var success = await blobService.CreateBlobsAsync(importNotificationBlobItems
- .Concat(alvsClearanceRequestBlobItems).ToArray());
+ var success = await blobService.CreateBlobsAsync(blobs.ToArray());
+
+ // var success = await blobService.CreateBlobsAsync(importNotificationBlobItems
+ // .Concat(alvsClearanceRequestBlobItems).ToArray());
return success;
}
diff --git a/TestDataGenerator/Helpers/DataHelpers.cs b/TestDataGenerator/Helpers/DataHelpers.cs
index 732f40f..92f1963 100644
--- a/TestDataGenerator/Helpers/DataHelpers.cs
+++ b/TestDataGenerator/Helpers/DataHelpers.cs
@@ -1,3 +1,4 @@
+using Btms.Common.Extensions;
using Btms.Model;
using Btms.Types.Ipaffs.V1.Extensions;
using Btms.Types.Alvs;
@@ -7,6 +8,21 @@ namespace TestDataGenerator.Helpers;
public static class DataHelpers
{
+ internal static string BlobPath(this object resource, string rootPath)
+ {
+ switch (resource)
+ {
+ case null:
+ throw new ArgumentNullException();
+ case ImportNotification n:
+ return n.BlobPath(rootPath);
+ case AlvsClearanceRequest cr:
+ return cr.BlobPath(rootPath);
+ default:
+ throw new InvalidDataException($"Unexpected type {resource.GetType().Name}");
+ }
+ }
+
internal static string BlobPath(this ImportNotification notification, string rootPath)
{
var dateString = notification.LastUpdated!.Value.ToString("yyyy/MM/dd");
@@ -22,9 +38,10 @@ internal static string DateRef(this DateTime created)
internal static string BlobPath(this AlvsClearanceRequest clearanceRequest, string rootPath)
{
var dateString = clearanceRequest.ServiceHeader!.ServiceCallTimestamp!.Value.ToString("yyyy/MM/dd");
-
+ var subPath = clearanceRequest.Header!.DecisionNumber.HasValue() ? "DECISIONS" : "ALVS";
+
return
- $"{rootPath}/ALVS/{dateString}/{clearanceRequest.Header!.EntryReference!.Replace(".", "")}-{Guid.NewGuid()}.json";
+ $"{rootPath}/{subPath}/{dateString}/{clearanceRequest.Header!.EntryReference!.Replace(".", "")}-{Guid.NewGuid()}.json";
}
internal static string AsCdsEntryReference(this MatchIdentifier identifier)
diff --git a/TestDataGenerator/ImportNotificationBuilder.cs b/TestDataGenerator/ImportNotificationBuilder.cs
index a3a8514..e5e0936 100644
--- a/TestDataGenerator/ImportNotificationBuilder.cs
+++ b/TestDataGenerator/ImportNotificationBuilder.cs
@@ -68,9 +68,14 @@ public ImportNotificationBuilder WithReferenceNumber(ImportNotificationTypeEn
x.ReferenceNumber = DataHelpers.GenerateReferenceNumber(chedType, scenario, created, item));
}
- public ImportNotificationBuilder WithEntryDate(DateTime entryDate)
+ public ImportNotificationBuilder WithCreationDate(DateTime entryDate, bool randomTime = true)
{
- return Do(x => x.LastUpdated = entryDate.RandomTime());
+ var entry = randomTime ?
+ // We don't want documents created in the future!
+ entryDate.RandomTime(entryDate.Date == DateTime.Today ? DateTime.Now.AddHours(-2).Hour : 23)
+ : entryDate;
+
+ return Do(x => x.LastUpdated = entry);
}
public ImportNotificationBuilder WithRandomArrivalDateTime(int maxDays, int maxHours=6)
diff --git a/TestDataGenerator/Program.cs b/TestDataGenerator/Program.cs
index c35f996..aa0e27c 100644
--- a/TestDataGenerator/Program.cs
+++ b/TestDataGenerator/Program.cs
@@ -66,7 +66,11 @@ private static async Task Main(string[] args)
{
Dataset = "LoadTest-One",
RootPath = "GENERATED-LOADTEST-ONE",
- Scenarios = new[] { app.CreateScenarioConfig(1, 3) }
+ Scenarios = new[]
+ {
+ app.CreateScenarioConfig(1, 1),
+ app.CreateScenarioConfig(1, 1)
+ }
},
new
{
diff --git a/TestDataGenerator/Properties/launchSettings.json b/TestDataGenerator/Properties/launchSettings.json
index 31c3391..b44e8bf 100644
--- a/TestDataGenerator/Properties/launchSettings.json
+++ b/TestDataGenerator/Properties/launchSettings.json
@@ -12,6 +12,17 @@
"AZURE_TENANT_ID": "c9d74090-b4e6-4b04-981d-e6757a160812"
}
},
+ "Generate LoadTest-One": {
+ "commandName": "Project",
+ "commandLineArgs": "LoadTest-One",
+ "environmentVariables": {
+ "DMP_ENVIRONMENT": "dev",
+ "DMP_SERVICE_BUS_NAME": "DEVTREINFSB1001",
+ "DMP_BLOB_STORAGE_NAME": "devdmpinfdl1001",
+ "DMP_SLOT": "1003",
+ "AZURE_TENANT_ID": "c9d74090-b4e6-4b04-981d-e6757a160812"
+ }
+ },
"Generate LoadTest-Basic": {
"commandName": "Project",
"commandLineArgs": "LoadTest-Basic",
diff --git a/TestDataGenerator/README.md b/TestDataGenerator/README.md
index 66411f6..257fe09 100644
--- a/TestDataGenerator/README.md
+++ b/TestDataGenerator/README.md
@@ -24,4 +24,12 @@ az storage blob upload-batch -d dmp-data-1001 --account-name snddmpinfdl1001 -s
TestDataGenerator/.test-data-generator/GENERATED-LOADTEST-90Dx10k --destination-path GENERATED-LOADTEST-90Dx10k
az storage blob upload-batch -d dmp-data-1001 --account-name snddmpinfdl1001 -s TestDataGenerator/.test-data-generator/GENERATED-LOADTEST-BASIC --destination-path GENERATED-LOADTEST-BASIC
-az storage blob upload-batch -d dmp-data-1001 --account-name snddmpinfdl1001 -s TestDataGenerator/.test-data-generator/PRODREDACTED-20241204 --destination-path PRODREDACTED-20241204
\ No newline at end of file
+az storage blob upload-batch -d dmp-data-1001 --account-name snddmpinfdl1001 -s TestDataGenerator/.test-data-generator/PRODREDACTED-20241204 --destination-path PRODREDACTED-20241204
+
+
+az storage blob directory list -c dmp-data-1001 -d --account-name snddmpinfdl1001
+
+az storage fs directory list -f dmp-data-1001 -d --account-name snddmpinfdl1001
+
+
+az storage fs directory download -f dmp-data-1001 -s "PRODREDACTED-20241204" -d "TestDataGenerator/.test-data-generator" --recursive --account-name snddmpinfdl1001
diff --git a/TestDataGenerator/ScenarioGenerator.cs b/TestDataGenerator/ScenarioGenerator.cs
index be15465..fe4aec7 100644
--- a/TestDataGenerator/ScenarioGenerator.cs
+++ b/TestDataGenerator/ScenarioGenerator.cs
@@ -1,6 +1,11 @@
+using System.Collections;
+using AutoFixture;
+using Btms.Common.Extensions;
+using Btms.Model;
using Btms.Types.Alvs;
using Btms.Types.Ipaffs;
using TestDataGenerator.Scenarios;
+using Decision = Btms.Types.Alvs.Decision;
namespace TestDataGenerator;
@@ -27,9 +32,62 @@ internal ClearanceRequestBuilder GetClearanceRequestBuilder(string file)
return builder;
}
- public class GeneratorResult
+ internal DecisionBuilder GetDecisionBuilder(string file)
{
- public ImportNotification[] ImportNotifications { get; set; } = default!;
- public AlvsClearanceRequest[] ClearanceRequests { get; set; } = default!;
+ var fullPath = $"{_fullFolder}/{file}.json";
+ var builder = new DecisionBuilder(fullPath);
+
+ return builder;
+ }
+
+ ///
+ /// A class to hold a list of message types we support. Would be nice to use something
+ /// other than object :|
+ ///
+ public class GeneratorResult : IEnumerable