From 07cd1ae2774756e78eb61b5f789c24a8dc0314b2 Mon Sep 17 00:00:00 2001 From: Craig Edmunds Date: Tue, 10 Dec 2024 12:13:06 +0000 Subject: [PATCH 1/5] CDMS-179 fixes errors when no matches found (#9) --- Btms.Analytics/ImportNotificationsAggregationService.cs | 5 ++++- Btms.Analytics/MovementsAggregationService.cs | 7 ++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/Btms.Analytics/ImportNotificationsAggregationService.cs b/Btms.Analytics/ImportNotificationsAggregationService.cs index 67345d9..b7cc613 100644 --- a/Btms.Analytics/ImportNotificationsAggregationService.cs +++ b/Btms.Analytics/ImportNotificationsAggregationService.cs @@ -77,7 +77,10 @@ public Task ByCommodityCount(DateTime from, DateTime to) .GroupBy(r => new { r.Key.ImportNotificationType, r.Key.Linked }) .ToList(); - var maxCommodities = result.Max(r => r.Max(i => i.Key.CommodityCount)); + // var maxCommodities = result.Max(r => r.Max(i => i.Key.CommodityCount)); + + var maxCommodities = result.Count > 0 ? + result.Max(r => r.Any() ? r.Max(i => i.Key.CommodityCount) : 0) : 0; var list = result .SelectMany(g => diff --git a/Btms.Analytics/MovementsAggregationService.cs b/Btms.Analytics/MovementsAggregationService.cs index 672fb26..7a4a9f8 100644 --- a/Btms.Analytics/MovementsAggregationService.cs +++ b/Btms.Analytics/MovementsAggregationService.cs @@ -74,8 +74,8 @@ public Task ByItemCount(DateTime from, DateTime to) var dictionary = mongoResult .ToDictionary(g => new { Title = AnalyticsHelpers.GetLinkedName(g.Linked), ItemCount = g.ItemCount }, g => g.Count); - var maxCount = mongoResult - .Max(r => r.Count); + var maxCount = mongoResult.Count > 0 ? + mongoResult.Max(r => r.Count) : 0; return Task.FromResult(AnalyticsHelpers.GetMovementSegments() .Select(title => new MultiSeriesDataset(title, "Item Count") { @@ -120,7 +120,8 @@ public Task ByUniqueDocumentReferenceCount(DateTime from, g => new { Title = AnalyticsHelpers.GetLinkedName(g.Linked), DocumentReferenceCount = g.DocumentReferenceCount }, g => g.MovementCount)!; - var maxReferences = mongoResult.Max(r => r.DocumentReferenceCount); + var maxReferences = mongoResult.Count > 0 ? + mongoResult.Max(r => r.DocumentReferenceCount) : 0; return Task.FromResult(AnalyticsHelpers.GetMovementSegments() .Select(title => new MultiSeriesDataset(title, "Document Reference Count") { From 70ab5ac023bf6c1b318c6677d5448d80505eb97c Mon Sep 17 00:00:00 2001 From: Craig Edmunds Date: Tue, 10 Dec 2024 13:29:50 +0000 Subject: [PATCH 2/5] CDMS-179 optimises Movements ByItemCount analytics for bigger datasets (#10) --- Btms.Analytics/MovementsAggregationService.cs | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/Btms.Analytics/MovementsAggregationService.cs b/Btms.Analytics/MovementsAggregationService.cs index 7a4a9f8..ce31078 100644 --- a/Btms.Analytics/MovementsAggregationService.cs +++ b/Btms.Analytics/MovementsAggregationService.cs @@ -13,6 +13,7 @@ using MongoDB.Driver; using Btms.Analytics.Extensions; +using Microsoft.EntityFrameworkCore.Query.SqlExpressions; namespace Btms.Analytics; @@ -58,17 +59,11 @@ public Task ByItemCount(DateTime from, DateTime to) var mongoQuery = context .Movements .Where(n => n.CreatedSource >= from && n.CreatedSource < to) - .GroupBy(m => new { Linked = m.Relationships.Notifications.Data.Count > 0, Items = m.Items.Count }) - .GroupBy(g => g.Key.Linked); - + .GroupBy(m => new { Linked = m.Relationships.Notifications.Data.Count > 0, ItemCount = m.Items.Count }) + .Select(g => new { g.Key.Linked, g.Key.ItemCount, Count = g.Count() }); + var mongoResult = mongoQuery .Execute(logger) - .SelectMany(g => g.Select(l => new - { - Linked = g.Key, - ItemCount = l.Key.Items, - Count = l.Count() - })) .ToList(); var dictionary = mongoResult From 315714c6f6c58472be65c164f55bff313cfbcfe7 Mon Sep 17 00:00:00 2001 From: Craig Edmunds Date: Tue, 10 Dec 2024 13:42:51 +0000 Subject: [PATCH 3/5] CDMS-179 optimises Movements ByItemCount analytics for bigger datasets (#11) --- Btms.Analytics/MovementsAggregationService.cs | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/Btms.Analytics/MovementsAggregationService.cs b/Btms.Analytics/MovementsAggregationService.cs index ce31078..4009d96 100644 --- a/Btms.Analytics/MovementsAggregationService.cs +++ b/Btms.Analytics/MovementsAggregationService.cs @@ -98,16 +98,10 @@ public Task ByUniqueDocumentReferenceCount(DateTime from, .Distinct() .Count() }) - .GroupBy(g => g.Key.Linked); - + .Select(g => new { g.Key.Linked, g.Key.DocumentReferenceCount, MovementCount = g.Count() }); + var mongoResult = mongoQuery .Execute(logger) - .SelectMany(g => g.Select(l => new - { - Linked = g.Key, - DocumentReferenceCount = l.Key.DocumentReferenceCount, - MovementCount = l.Count() - })) .ToList(); var dictionary = mongoResult From 64c8d7cf265f5cce350dde6e152c808aa0d7d56b Mon Sep 17 00:00:00 2001 From: Tris Date: Tue, 10 Dec 2024 14:05:12 +0000 Subject: [PATCH 4/5] rearranged domain services, added decisions skeleton --- .../Decisions/DecisionServiceTests.cs | 19 ++++++ .../Finders/ChedADecisionFinderTests.cs | 19 ++++++ .../Finders/ChedDDecisionFinderTests.cs | 19 ++++++ .../Finders/ChedPDecisionFinderTests.cs | 19 ++++++ .../Finders/ChedPPDecisionFinderTests.cs | 19 ++++++ .../{ => Linking}/LinkingServiceTests.cs | 4 +- Btms.Business/Btms.Business.csproj | 4 ++ .../Extensions/ServiceCollectionExtensions.cs | 4 +- .../Services/Decisions/DecisionCode.cs | 15 +++++ .../Services/Decisions/DecisionContext.cs | 10 ++++ .../Services/Decisions/DecisionResult.cs | 3 + .../Services/Decisions/DecisionService.cs | 58 +++++++++++++++++++ .../Decisions/Finders/ChedADecisionFinder.cs | 11 ++++ .../Decisions/Finders/ChedDDecisionFinder.cs | 11 ++++ .../Decisions/Finders/ChedPDecisionFinder.cs | 11 ++++ .../Decisions/Finders/ChedPPDecisionFinder.cs | 11 ++++ .../Decisions/Finders/IDecisionFinder.cs | 8 +++ .../Services/Decisions/IDecisionService.cs | 6 ++ .../Services/{ => Linking}/ILinkingService.cs | 2 +- .../ImportNotificationLinkContext.cs | 2 +- .../Services/{ => Linking}/LinkContext.cs | 2 +- .../Services/{ => Linking}/LinkException.cs | 2 +- .../Services/{ => Linking}/LinkOutcome.cs | 2 +- .../Services/{ => Linking}/LinkResult.cs | 2 +- .../Services/{ => Linking}/LinkingService.cs | 4 +- .../{ => Linking}/MovementLinkContext.cs | 2 +- .../ClearanceRequestConsumerTests.cs | 2 +- .../NotificationsConsumerTests.cs | 3 +- .../AlvsClearanceRequestConsumer.cs | 2 +- Btms.Consumers/NotificationConsumer.cs | 2 +- 30 files changed, 260 insertions(+), 18 deletions(-) create mode 100644 Btms.Business.Tests/Services/Decisions/DecisionServiceTests.cs create mode 100644 Btms.Business.Tests/Services/Decisions/Finders/ChedADecisionFinderTests.cs create mode 100644 Btms.Business.Tests/Services/Decisions/Finders/ChedDDecisionFinderTests.cs create mode 100644 Btms.Business.Tests/Services/Decisions/Finders/ChedPDecisionFinderTests.cs create mode 100644 Btms.Business.Tests/Services/Decisions/Finders/ChedPPDecisionFinderTests.cs rename Btms.Business.Tests/Services/{ => Linking}/LinkingServiceTests.cs (99%) create mode 100644 Btms.Business/Services/Decisions/DecisionCode.cs create mode 100644 Btms.Business/Services/Decisions/DecisionContext.cs create mode 100644 Btms.Business/Services/Decisions/DecisionResult.cs create mode 100644 Btms.Business/Services/Decisions/DecisionService.cs create mode 100644 Btms.Business/Services/Decisions/Finders/ChedADecisionFinder.cs create mode 100644 Btms.Business/Services/Decisions/Finders/ChedDDecisionFinder.cs create mode 100644 Btms.Business/Services/Decisions/Finders/ChedPDecisionFinder.cs create mode 100644 Btms.Business/Services/Decisions/Finders/ChedPPDecisionFinder.cs create mode 100644 Btms.Business/Services/Decisions/Finders/IDecisionFinder.cs create mode 100644 Btms.Business/Services/Decisions/IDecisionService.cs rename Btms.Business/Services/{ => Linking}/ILinkingService.cs (76%) rename Btms.Business/Services/{ => Linking}/ImportNotificationLinkContext.cs (87%) rename Btms.Business/Services/{ => Linking}/LinkContext.cs (93%) rename Btms.Business/Services/{ => Linking}/LinkException.cs (66%) rename Btms.Business/Services/{ => Linking}/LinkOutcome.cs (56%) rename Btms.Business/Services/{ => Linking}/LinkResult.cs (86%) rename Btms.Business/Services/{ => Linking}/LinkingService.cs (99%) rename Btms.Business/Services/{ => Linking}/MovementLinkContext.cs (86%) diff --git a/Btms.Business.Tests/Services/Decisions/DecisionServiceTests.cs b/Btms.Business.Tests/Services/Decisions/DecisionServiceTests.cs new file mode 100644 index 0000000..6433285 --- /dev/null +++ b/Btms.Business.Tests/Services/Decisions/DecisionServiceTests.cs @@ -0,0 +1,19 @@ +using Btms.Business.Services.Decisions; +using Xunit; + +namespace Btms.Business.Tests.Services.Decisions; + +public class DecisionServiceTests +{ + [Fact] + public async Task DeriveDecision() + { + // Arrange + var sut = new DecisionService(); + + // Act + + // Assert + await Task.CompletedTask; + } +} \ No newline at end of file diff --git a/Btms.Business.Tests/Services/Decisions/Finders/ChedADecisionFinderTests.cs b/Btms.Business.Tests/Services/Decisions/Finders/ChedADecisionFinderTests.cs new file mode 100644 index 0000000..afba8cc --- /dev/null +++ b/Btms.Business.Tests/Services/Decisions/Finders/ChedADecisionFinderTests.cs @@ -0,0 +1,19 @@ +using Btms.Business.Services.Decisions.Finders; +using Xunit; + +namespace Btms.Business.Tests.Services.Decisions.Finders; + +public class ChedADecisionFinderTests +{ + [Fact] + public async Task Find_Should_DoThings() + { + // Arrange + var sut = new ChedADecisionFinder(); + + // Act + + // Assert + await Task.CompletedTask; + } +} \ No newline at end of file diff --git a/Btms.Business.Tests/Services/Decisions/Finders/ChedDDecisionFinderTests.cs b/Btms.Business.Tests/Services/Decisions/Finders/ChedDDecisionFinderTests.cs new file mode 100644 index 0000000..323a9f3 --- /dev/null +++ b/Btms.Business.Tests/Services/Decisions/Finders/ChedDDecisionFinderTests.cs @@ -0,0 +1,19 @@ +using Btms.Business.Services.Decisions.Finders; +using Xunit; + +namespace Btms.Business.Tests.Services.Decisions.Finders; + +public class ChedDDecisionFinderTests +{ + [Fact] + public async Task Find_Should_DoThings() + { + // Arrange + var sut = new ChedDDecisionFinder(); + + // Act + + // Assert + await Task.CompletedTask; + } +} \ No newline at end of file diff --git a/Btms.Business.Tests/Services/Decisions/Finders/ChedPDecisionFinderTests.cs b/Btms.Business.Tests/Services/Decisions/Finders/ChedPDecisionFinderTests.cs new file mode 100644 index 0000000..e49d7d0 --- /dev/null +++ b/Btms.Business.Tests/Services/Decisions/Finders/ChedPDecisionFinderTests.cs @@ -0,0 +1,19 @@ +using Btms.Business.Services.Decisions.Finders; +using Xunit; + +namespace Btms.Business.Tests.Services.Decisions.Finders; + +public class ChedPDecisionFinderTests +{ + [Fact] + public async Task Find_Should_DoThings() + { + // Arrange + var sut = new ChedPDecisionFinder(); + + // Act + + // Assert + await Task.CompletedTask; + } +} \ No newline at end of file diff --git a/Btms.Business.Tests/Services/Decisions/Finders/ChedPPDecisionFinderTests.cs b/Btms.Business.Tests/Services/Decisions/Finders/ChedPPDecisionFinderTests.cs new file mode 100644 index 0000000..c242a76 --- /dev/null +++ b/Btms.Business.Tests/Services/Decisions/Finders/ChedPPDecisionFinderTests.cs @@ -0,0 +1,19 @@ +using Btms.Business.Services.Decisions.Finders; +using Xunit; + +namespace Btms.Business.Tests.Services.Decisions.Finders; + +public class ChedPPDecisionFinderTests +{ + [Fact] + public async Task Find_Should_DoThings() + { + // Arrange + var sut = new ChedPPDecisionFinder(); + + // Act + + // Assert + await Task.CompletedTask; + } +} \ No newline at end of file diff --git a/Btms.Business.Tests/Services/LinkingServiceTests.cs b/Btms.Business.Tests/Services/Linking/LinkingServiceTests.cs similarity index 99% rename from Btms.Business.Tests/Services/LinkingServiceTests.cs rename to Btms.Business.Tests/Services/Linking/LinkingServiceTests.cs index ace93c7..ce8e665 100644 --- a/Btms.Business.Tests/Services/LinkingServiceTests.cs +++ b/Btms.Business.Tests/Services/Linking/LinkingServiceTests.cs @@ -1,6 +1,6 @@ using Btms.Backend.Data; using Btms.Backend.Data.InMemory; -using Btms.Business.Services; +using Btms.Business.Services.Linking; using Btms.Metrics; using Btms.Model.Alvs; using Btms.Model.ChangeLog; @@ -12,7 +12,7 @@ using Items = Btms.Model.Alvs.Items; using Movement = Btms.Model.Movement; -namespace Btms.Business.Tests.Services; +namespace Btms.Business.Tests.Services.Linking; public class LinkingServiceTests { diff --git a/Btms.Business/Btms.Business.csproj b/Btms.Business/Btms.Business.csproj index 0057217..2e59068 100644 --- a/Btms.Business/Btms.Business.csproj +++ b/Btms.Business/Btms.Business.csproj @@ -35,4 +35,8 @@ + + + + diff --git a/Btms.Business/Extensions/ServiceCollectionExtensions.cs b/Btms.Business/Extensions/ServiceCollectionExtensions.cs index f68acb7..095c29e 100644 --- a/Btms.Business/Extensions/ServiceCollectionExtensions.cs +++ b/Btms.Business/Extensions/ServiceCollectionExtensions.cs @@ -2,7 +2,6 @@ using Btms.Business.Pipelines; using Btms.Business.Pipelines.Matching; using Btms.Business.Pipelines.Matching.Rules; -using Btms.Business.Services; using Btms.Backend.Data.Extensions; using Btms.BlobService; using Btms.BlobService.Extensions; @@ -14,6 +13,8 @@ using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Btms.Business.Pipelines.PreProcessing; +using Btms.Business.Services.Decisions; +using Btms.Business.Services.Linking; using Btms.Types.Alvs; namespace Btms.Business.Extensions @@ -61,6 +62,7 @@ public static IServiceCollection AddBusinessServices(this IServiceCollection ser }); services.AddScoped(); + services.AddScoped(); services.AddScoped, ImportNotificationPreProcessor>(); services.AddScoped, MovementPreProcessor>(); diff --git a/Btms.Business/Services/Decisions/DecisionCode.cs b/Btms.Business/Services/Decisions/DecisionCode.cs new file mode 100644 index 0000000..7f916bd --- /dev/null +++ b/Btms.Business/Services/Decisions/DecisionCode.cs @@ -0,0 +1,15 @@ +namespace Btms.Business.Services.Decisions; + +public enum DecisionCode +{ + E03, + C03, + C05, + C06, + C07, + C08, + N02, + N03, + N04, + N07 +} \ No newline at end of file diff --git a/Btms.Business/Services/Decisions/DecisionContext.cs b/Btms.Business/Services/Decisions/DecisionContext.cs new file mode 100644 index 0000000..f4a5d87 --- /dev/null +++ b/Btms.Business/Services/Decisions/DecisionContext.cs @@ -0,0 +1,10 @@ +using Btms.Model; +using Btms.Model.Ipaffs; + +namespace Btms.Business.Services.Decisions; + +public class DecisionContext +{ + public List Notifications { get; set; } = new(); + public List Movements { get; set; } = new(); +} \ No newline at end of file diff --git a/Btms.Business/Services/Decisions/DecisionResult.cs b/Btms.Business/Services/Decisions/DecisionResult.cs new file mode 100644 index 0000000..67f02dd --- /dev/null +++ b/Btms.Business/Services/Decisions/DecisionResult.cs @@ -0,0 +1,3 @@ +namespace Btms.Business.Services.Decisions; + +public record DecisionResult(DecisionCode DecisionCode); \ No newline at end of file diff --git a/Btms.Business/Services/Decisions/DecisionService.cs b/Btms.Business/Services/Decisions/DecisionService.cs new file mode 100644 index 0000000..800441f --- /dev/null +++ b/Btms.Business/Services/Decisions/DecisionService.cs @@ -0,0 +1,58 @@ +using Btms.Business.Services.Decisions.Finders; +using Btms.Model.Ipaffs; + +namespace Btms.Business.Services.Decisions; + +public class DecisionService : IDecisionService +{ + public async Task DeriveDecision(DecisionContext decisionContext) + { + // Validate and prerequisite checks + + var decisions = decisionContext.Notifications.Select(x => (x.Id, GetDecision(x))).ToList(); + + foreach (var movement in decisionContext.Movements) + { + // Generate list matched items -> decisions + + foreach (var item in movement.Items) + { + // check decisions list for match reference, if no match then drop out with "no-match" + + } + + // Return decision based on prioritisation from confluence + } + + await Task.CompletedTask; + return new DecisionResult(DecisionCode.N02); + } + + private DecisionResult GetDecision(ImportNotification notification) + { + // get decision finder - fold IUU stuff into the decision finder for fish? + IDecisionFinder finder; + switch (notification.ImportNotificationType) + { + case ImportNotificationTypeEnum.Ced: + finder = new ChedDDecisionFinder(); + break; + + case ImportNotificationTypeEnum.Cveda: + finder = new ChedADecisionFinder(); + break; + + case ImportNotificationTypeEnum.Cvedp: + finder = new ChedPDecisionFinder(); + break; + + case ImportNotificationTypeEnum.Chedpp: + finder = new ChedPPDecisionFinder(); + break; + + default: throw new InvalidOperationException("Invalid ImportNotificationType"); + } + + return finder.FindDecision(notification); + } +} \ No newline at end of file diff --git a/Btms.Business/Services/Decisions/Finders/ChedADecisionFinder.cs b/Btms.Business/Services/Decisions/Finders/ChedADecisionFinder.cs new file mode 100644 index 0000000..b9b2c37 --- /dev/null +++ b/Btms.Business/Services/Decisions/Finders/ChedADecisionFinder.cs @@ -0,0 +1,11 @@ +using Btms.Model.Ipaffs; + +namespace Btms.Business.Services.Decisions.Finders; + +public class ChedADecisionFinder : IDecisionFinder +{ + public DecisionResult FindDecision(ImportNotification notification) + { + throw new NotImplementedException(); + } +} \ No newline at end of file diff --git a/Btms.Business/Services/Decisions/Finders/ChedDDecisionFinder.cs b/Btms.Business/Services/Decisions/Finders/ChedDDecisionFinder.cs new file mode 100644 index 0000000..3d289f4 --- /dev/null +++ b/Btms.Business/Services/Decisions/Finders/ChedDDecisionFinder.cs @@ -0,0 +1,11 @@ +using Btms.Model.Ipaffs; + +namespace Btms.Business.Services.Decisions.Finders; + +public class ChedDDecisionFinder : IDecisionFinder +{ + public DecisionResult FindDecision(ImportNotification notification) + { + throw new NotImplementedException(); + } +} \ No newline at end of file diff --git a/Btms.Business/Services/Decisions/Finders/ChedPDecisionFinder.cs b/Btms.Business/Services/Decisions/Finders/ChedPDecisionFinder.cs new file mode 100644 index 0000000..3f6bf36 --- /dev/null +++ b/Btms.Business/Services/Decisions/Finders/ChedPDecisionFinder.cs @@ -0,0 +1,11 @@ +using Btms.Model.Ipaffs; + +namespace Btms.Business.Services.Decisions.Finders; + +public class ChedPDecisionFinder : IDecisionFinder +{ + public DecisionResult FindDecision(ImportNotification notification) + { + throw new NotImplementedException(); + } +} \ No newline at end of file diff --git a/Btms.Business/Services/Decisions/Finders/ChedPPDecisionFinder.cs b/Btms.Business/Services/Decisions/Finders/ChedPPDecisionFinder.cs new file mode 100644 index 0000000..c9f4265 --- /dev/null +++ b/Btms.Business/Services/Decisions/Finders/ChedPPDecisionFinder.cs @@ -0,0 +1,11 @@ +using Btms.Model.Ipaffs; + +namespace Btms.Business.Services.Decisions.Finders; + +public class ChedPPDecisionFinder : IDecisionFinder +{ + public DecisionResult FindDecision(ImportNotification notification) + { + throw new NotImplementedException(); + } +} \ No newline at end of file diff --git a/Btms.Business/Services/Decisions/Finders/IDecisionFinder.cs b/Btms.Business/Services/Decisions/Finders/IDecisionFinder.cs new file mode 100644 index 0000000..1863489 --- /dev/null +++ b/Btms.Business/Services/Decisions/Finders/IDecisionFinder.cs @@ -0,0 +1,8 @@ +using Btms.Model.Ipaffs; + +namespace Btms.Business.Services.Decisions.Finders; + +public interface IDecisionFinder +{ + DecisionResult FindDecision(ImportNotification notification); +} \ No newline at end of file diff --git a/Btms.Business/Services/Decisions/IDecisionService.cs b/Btms.Business/Services/Decisions/IDecisionService.cs new file mode 100644 index 0000000..c1483ae --- /dev/null +++ b/Btms.Business/Services/Decisions/IDecisionService.cs @@ -0,0 +1,6 @@ +namespace Btms.Business.Services.Decisions; + +public interface IDecisionService +{ + public Task DeriveDecision(DecisionContext decisionContext); +} \ No newline at end of file diff --git a/Btms.Business/Services/ILinkingService.cs b/Btms.Business/Services/Linking/ILinkingService.cs similarity index 76% rename from Btms.Business/Services/ILinkingService.cs rename to Btms.Business/Services/Linking/ILinkingService.cs index b918b92..61482f6 100644 --- a/Btms.Business/Services/ILinkingService.cs +++ b/Btms.Business/Services/Linking/ILinkingService.cs @@ -1,5 +1,5 @@ -namespace Btms.Business.Services; +namespace Btms.Business.Services.Linking; public interface ILinkingService { diff --git a/Btms.Business/Services/ImportNotificationLinkContext.cs b/Btms.Business/Services/Linking/ImportNotificationLinkContext.cs similarity index 87% rename from Btms.Business/Services/ImportNotificationLinkContext.cs rename to Btms.Business/Services/Linking/ImportNotificationLinkContext.cs index d8b3821..ab482bb 100644 --- a/Btms.Business/Services/ImportNotificationLinkContext.cs +++ b/Btms.Business/Services/Linking/ImportNotificationLinkContext.cs @@ -1,7 +1,7 @@ using Btms.Model.ChangeLog; using Btms.Model.Ipaffs; -namespace Btms.Business.Services; +namespace Btms.Business.Services.Linking; public record ImportNotificationLinkContext(ImportNotification PersistedImportNotification, ChangeSet? ChangeSet) : LinkContext { diff --git a/Btms.Business/Services/LinkContext.cs b/Btms.Business/Services/Linking/LinkContext.cs similarity index 93% rename from Btms.Business/Services/LinkContext.cs rename to Btms.Business/Services/Linking/LinkContext.cs index 93b5be8..e996c05 100644 --- a/Btms.Business/Services/LinkContext.cs +++ b/Btms.Business/Services/Linking/LinkContext.cs @@ -2,7 +2,7 @@ using Btms.Model.ChangeLog; using Btms.Model.Ipaffs; -namespace Btms.Business.Services; +namespace Btms.Business.Services.Linking; public abstract record LinkContext { diff --git a/Btms.Business/Services/LinkException.cs b/Btms.Business/Services/Linking/LinkException.cs similarity index 66% rename from Btms.Business/Services/LinkException.cs rename to Btms.Business/Services/Linking/LinkException.cs index c820f12..23c5014 100644 --- a/Btms.Business/Services/LinkException.cs +++ b/Btms.Business/Services/Linking/LinkException.cs @@ -1,3 +1,3 @@ -namespace Btms.Business.Services; +namespace Btms.Business.Services.Linking; public class LinkException(Exception inner) : Exception("Failed to link", inner); \ No newline at end of file diff --git a/Btms.Business/Services/LinkOutcome.cs b/Btms.Business/Services/Linking/LinkOutcome.cs similarity index 56% rename from Btms.Business/Services/LinkOutcome.cs rename to Btms.Business/Services/Linking/LinkOutcome.cs index 7507e58..d4922b6 100644 --- a/Btms.Business/Services/LinkOutcome.cs +++ b/Btms.Business/Services/Linking/LinkOutcome.cs @@ -1,4 +1,4 @@ -namespace Btms.Business.Services; +namespace Btms.Business.Services.Linking; public enum LinkOutcome { diff --git a/Btms.Business/Services/LinkResult.cs b/Btms.Business/Services/Linking/LinkResult.cs similarity index 86% rename from Btms.Business/Services/LinkResult.cs rename to Btms.Business/Services/Linking/LinkResult.cs index acd48fc..9f6b3ad 100644 --- a/Btms.Business/Services/LinkResult.cs +++ b/Btms.Business/Services/Linking/LinkResult.cs @@ -1,7 +1,7 @@ using Btms.Model; using Btms.Model.Ipaffs; -namespace Btms.Business.Services; +namespace Btms.Business.Services.Linking; public class LinkResult(LinkOutcome state) { diff --git a/Btms.Business/Services/LinkingService.cs b/Btms.Business/Services/Linking/LinkingService.cs similarity index 99% rename from Btms.Business/Services/LinkingService.cs rename to Btms.Business/Services/Linking/LinkingService.cs index 1c7e671..025bd14 100644 --- a/Btms.Business/Services/LinkingService.cs +++ b/Btms.Business/Services/Linking/LinkingService.cs @@ -1,16 +1,14 @@ using System.Text.RegularExpressions; using Btms.Backend.Data; using Btms.Backend.Data.Extensions; -using Btms.Common.Extensions; using Btms.Metrics; using Btms.Model; using Btms.Model.ChangeLog; using Btms.Model.Ipaffs; using Btms.Model.Relationships; -using Json.Patch; using Microsoft.Extensions.Logging; -namespace Btms.Business.Services; +namespace Btms.Business.Services.Linking; public static partial class LinkingServiceLogging { diff --git a/Btms.Business/Services/MovementLinkContext.cs b/Btms.Business/Services/Linking/MovementLinkContext.cs similarity index 86% rename from Btms.Business/Services/MovementLinkContext.cs rename to Btms.Business/Services/Linking/MovementLinkContext.cs index e55fe31..0826c6d 100644 --- a/Btms.Business/Services/MovementLinkContext.cs +++ b/Btms.Business/Services/Linking/MovementLinkContext.cs @@ -1,7 +1,7 @@ using Btms.Model; using Btms.Model.ChangeLog; -namespace Btms.Business.Services; +namespace Btms.Business.Services.Linking; public record MovementLinkContext(Movement PersistedMovement, ChangeSet? ChangeSet) : LinkContext { diff --git a/Btms.Consumers.Tests/ClearanceRequestConsumerTests.cs b/Btms.Consumers.Tests/ClearanceRequestConsumerTests.cs index 6320736..42aaa3b 100644 --- a/Btms.Consumers.Tests/ClearanceRequestConsumerTests.cs +++ b/Btms.Consumers.Tests/ClearanceRequestConsumerTests.cs @@ -1,5 +1,5 @@ using Btms.Business.Pipelines.PreProcessing; -using Btms.Business.Services; +using Btms.Business.Services.Linking; using Btms.Consumers.Extensions; using Btms.Model; using Btms.Model.Auditing; diff --git a/Btms.Consumers.Tests/NotificationsConsumerTests.cs b/Btms.Consumers.Tests/NotificationsConsumerTests.cs index 2dd4419..37a17b4 100644 --- a/Btms.Consumers.Tests/NotificationsConsumerTests.cs +++ b/Btms.Consumers.Tests/NotificationsConsumerTests.cs @@ -1,6 +1,5 @@ using Btms.Business.Pipelines.PreProcessing; -using Btms.Business.Services; -using Btms.Consumers; +using Btms.Business.Services.Linking; using Btms.Consumers.Extensions; using Btms.Model.Auditing; using Btms.Types.Ipaffs; diff --git a/Btms.Consumers/AlvsClearanceRequestConsumer.cs b/Btms.Consumers/AlvsClearanceRequestConsumer.cs index 8a7550e..b96c2c7 100644 --- a/Btms.Consumers/AlvsClearanceRequestConsumer.cs +++ b/Btms.Consumers/AlvsClearanceRequestConsumer.cs @@ -1,9 +1,9 @@ -using Btms.Business.Services; using Btms.Types.Alvs; using Microsoft.Extensions.Logging; using SlimMessageBus; using Btms.Consumers.Extensions; using Btms.Business.Pipelines.PreProcessing; +using Btms.Business.Services.Linking; namespace Btms.Consumers { diff --git a/Btms.Consumers/NotificationConsumer.cs b/Btms.Consumers/NotificationConsumer.cs index 66c08c2..72e99c2 100644 --- a/Btms.Consumers/NotificationConsumer.cs +++ b/Btms.Consumers/NotificationConsumer.cs @@ -1,9 +1,9 @@ -using Btms.Business.Services; using Btms.Types.Ipaffs; using SlimMessageBus; using Btms.Consumers.Extensions; using Microsoft.Extensions.Logging; using Btms.Business.Pipelines.PreProcessing; +using Btms.Business.Services.Linking; namespace Btms.Consumers { From 2c10d514113262be84ababd8b3a61b6fafb57656 Mon Sep 17 00:00:00 2001 From: Craig Edmunds Date: Tue, 10 Dec 2024 16:25:57 +0000 Subject: [PATCH 5/5] Feature/cdms 188 configure concurrency (#15) * CDMS-188 setting concurency * CDMS-188 options for concurrency working * Fixed test --- .../SyncClearanceRequestsCommandTests.cs | 2 +- Btms.Business/BusinessOptions.cs | 39 ++++++++++++++++++- .../Commands/SyncClearanceRequestsCommand.cs | 6 +-- .../Commands/SyncDecisionsCommand.cs | 4 +- Btms.Business/Commands/SyncGmrsCommand.cs | 4 +- Btms.Business/Commands/SyncHandler.cs | 21 ++++++---- .../Commands/SyncNotificationsCommand.cs | 4 +- Btms.Consumers/ConsumerOptions.cs | 15 +++++++ .../Extensions/ServiceCollectionExtensions.cs | 24 +++++++++--- 9 files changed, 96 insertions(+), 23 deletions(-) create mode 100644 Btms.Consumers/ConsumerOptions.cs diff --git a/Btms.Business.Tests/Commands/SyncClearanceRequestsCommandTests.cs b/Btms.Business.Tests/Commands/SyncClearanceRequestsCommandTests.cs index 85c2621..63f3a89 100644 --- a/Btms.Business.Tests/Commands/SyncClearanceRequestsCommandTests.cs +++ b/Btms.Business.Tests/Commands/SyncClearanceRequestsCommandTests.cs @@ -51,7 +51,7 @@ public async Task WhenClearanceRequestBlobsExist_ThenTheyShouldBePlacedOnInterna await handler.Handle(command, CancellationToken.None); // ASSERT - await bus.Received(1).Publish(Arg.Any(), "ALVS", + await bus.Received(1).Publish(Arg.Any(), "CLEARANCEREQUESTS", Arg.Any>(), Arg.Any()); } } diff --git a/Btms.Business/BusinessOptions.cs b/Btms.Business/BusinessOptions.cs index a62a3bd..30de4f1 100644 --- a/Btms.Business/BusinessOptions.cs +++ b/Btms.Business/BusinessOptions.cs @@ -1,5 +1,6 @@ using System.ComponentModel.DataAnnotations; using Btms.Azure; +using Btms.Business.Commands; namespace Btms.Business; @@ -7,6 +8,42 @@ public class BusinessOptions { public const string SectionName = nameof(BusinessOptions); - [Required] public string DmpBlobRootFolder { get; set; } = "RAW"; + private readonly int defaultDegreeOfParallelism = Math.Max(Environment.ProcessorCount / 4, 1); + [Required] public string DmpBlobRootFolder { get; set; } = "RAW"; + + public Dictionary> ConcurrencyConfiguration { get; set; } + + public enum Feature + { + BlobPaths, + BlobItems + } + public BusinessOptions() + { + ConcurrencyConfiguration = new Dictionary> + { + { + nameof(SyncNotificationsCommand), new Dictionary() + { + { Feature.BlobPaths, defaultDegreeOfParallelism }, { Feature.BlobItems, defaultDegreeOfParallelism } + } + }, + { + nameof(SyncClearanceRequestsCommand), new Dictionary() + { + { Feature.BlobPaths, defaultDegreeOfParallelism }, { Feature.BlobItems, defaultDegreeOfParallelism } + } + } + }; + } + + public int GetConcurrency(Feature feature) + { + if (ConcurrencyConfiguration.TryGetValue(typeof(T).Name, out var degreeOfParallelismDictionary)) + { + return degreeOfParallelismDictionary.GetValueOrDefault(feature, defaultDegreeOfParallelism); + } + return defaultDegreeOfParallelism; + } } \ No newline at end of file diff --git a/Btms.Business/Commands/SyncClearanceRequestsCommand.cs b/Btms.Business/Commands/SyncClearanceRequestsCommand.cs index f84431c..0cdf415 100644 --- a/Btms.Business/Commands/SyncClearanceRequestsCommand.cs +++ b/Btms.Business/Commands/SyncClearanceRequestsCommand.cs @@ -20,14 +20,14 @@ internal class Handler( IOptions businessOptions, ISyncJobStore syncJobStore) : SyncCommand.Handler(syncMetrics, bus, logger, sensitiveDataSerializer, - blobService, syncJobStore) + blobService, businessOptions, syncJobStore) { public override async Task Handle(SyncClearanceRequestsCommand request, CancellationToken cancellationToken) { var rootFolder = string.IsNullOrEmpty(request.RootFolder) - ? businessOptions.Value.DmpBlobRootFolder + ? Options.DmpBlobRootFolder : request.RootFolder; - await SyncBlobPaths(request.SyncPeriod, "ALVS", request.JobId, cancellationToken,$"{rootFolder}/ALVS"); + await SyncBlobPaths(request.SyncPeriod, "CLEARANCEREQUESTS", request.JobId, cancellationToken,$"{rootFolder}/ALVS"); } } diff --git a/Btms.Business/Commands/SyncDecisionsCommand.cs b/Btms.Business/Commands/SyncDecisionsCommand.cs index a549a9f..5c8f94e 100644 --- a/Btms.Business/Commands/SyncDecisionsCommand.cs +++ b/Btms.Business/Commands/SyncDecisionsCommand.cs @@ -19,12 +19,12 @@ internal class Handler( IBlobService blobService, IOptions businessOptions, ISyncJobStore syncJobStore) - : SyncCommand.Handler(syncMetrics, bus, logger, sensitiveDataSerializer, blobService, syncJobStore) + : SyncCommand.Handler(syncMetrics, bus, logger, sensitiveDataSerializer, blobService, businessOptions, syncJobStore) { public override async Task Handle(SyncDecisionsCommand request, CancellationToken cancellationToken) { var rootFolder = string.IsNullOrEmpty(request.RootFolder) - ? businessOptions.Value.DmpBlobRootFolder + ? Options.DmpBlobRootFolder : request.RootFolder; await SyncBlobPaths(request.SyncPeriod, "DECISIONS", request.JobId, cancellationToken,$"{rootFolder}/DECISIONS"); diff --git a/Btms.Business/Commands/SyncGmrsCommand.cs b/Btms.Business/Commands/SyncGmrsCommand.cs index d2fc0e6..be38c66 100644 --- a/Btms.Business/Commands/SyncGmrsCommand.cs +++ b/Btms.Business/Commands/SyncGmrsCommand.cs @@ -19,12 +19,12 @@ internal class Handler( IBlobService blobService, IOptions businessOptions, ISyncJobStore syncJobStore) - : SyncCommand.Handler(syncMetrics, bus, logger, sensitiveDataSerializer, blobService, syncJobStore) + : SyncCommand.Handler(syncMetrics, bus, logger, sensitiveDataSerializer, blobService, businessOptions, syncJobStore) { public override async Task Handle(SyncGmrsCommand request, CancellationToken cancellationToken) { var rootFolder = string.IsNullOrEmpty(request.RootFolder) - ? businessOptions.Value.DmpBlobRootFolder + ? Options.DmpBlobRootFolder : request.RootFolder; await SyncBlobPaths(request.SyncPeriod, "GMR", request.JobId, cancellationToken, $"{rootFolder}/GVMSAPIRESPONSE"); diff --git a/Btms.Business/Commands/SyncHandler.cs b/Btms.Business/Commands/SyncHandler.cs index b5260e8..a18dded 100644 --- a/Btms.Business/Commands/SyncHandler.cs +++ b/Btms.Business/Commands/SyncHandler.cs @@ -6,6 +6,7 @@ using System.Diagnostics; using System.Text.Json.Serialization; using Btms.Metrics; +using Microsoft.Extensions.Options; using IRequest = MediatR.IRequest; namespace Btms.Business.Commands; @@ -51,12 +52,14 @@ internal abstract class Handler( ILogger logger, ISensitiveDataSerializer sensitiveDataSerializer, IBlobService blobService, + IOptions options, ISyncJobStore syncJobStore) : MediatR.IRequestHandler where T : IRequest { - private readonly int maxDegreeOfParallelism = Math.Max(Environment.ProcessorCount / 4, 1); - + // private readonly int defaultDegreeOfParallelism = Math.Max(Environment.ProcessorCount / 4, 1); + protected readonly BusinessOptions Options = options.Value; + public const string ActivityName = "Btms.ProcessBlob"; public abstract Task Handle(T request, CancellationToken cancellationToken); @@ -65,20 +68,21 @@ protected async Task SyncBlobPaths(SyncPeriod period, string topic, Gu { var job = syncJobStore.GetJob(jobId); job?.Start(); + var degreeOfParallelism = options.Value.GetConcurrency(BusinessOptions.Feature.BlobPaths); using (logger.BeginScope(new List> { new("JobId", job?.JobId!), new("SyncPeriod", period.ToString()), - new("Parallelism", maxDegreeOfParallelism), + new("Parallelism", degreeOfParallelism), new("ProcessorCount", Environment.ProcessorCount), new("Command", typeof(T).Name), })) { - logger.SyncStarted(job?.JobId.ToString()!, period.ToString(), maxDegreeOfParallelism, Environment.ProcessorCount, typeof(T).Name); + logger.SyncStarted(job?.JobId.ToString()!, period.ToString(), degreeOfParallelism, Environment.ProcessorCount, typeof(T).Name); try { await Parallel.ForEachAsync(paths, - new ParallelOptions() { MaxDegreeOfParallelism = maxDegreeOfParallelism }, + new ParallelOptions() { MaxDegreeOfParallelism = degreeOfParallelism }, async (path, token) => { using (logger.BeginScope(new List> { new("SyncPath", path), })) @@ -104,8 +108,9 @@ protected async Task SyncBlobPath(string path, SyncPeriod period, stri CancellationToken cancellationToken) { var result = blobService.GetResourcesAsync($"{path}{period.GetPeriodPath()}", cancellationToken); + var degreeOfParallelism = options.Value.GetConcurrency(BusinessOptions.Feature.BlobItems); - await Parallel.ForEachAsync(result, new ParallelOptions() { CancellationToken = cancellationToken, MaxDegreeOfParallelism = maxDegreeOfParallelism }, async (item, token) => + await Parallel.ForEachAsync(result, new ParallelOptions() { CancellationToken = cancellationToken, MaxDegreeOfParallelism = degreeOfParallelism }, async (item, token) => { await SyncBlob(path, topic, item, job, cancellationToken); }); @@ -115,8 +120,10 @@ protected async Task SyncBlobPath(string path, SyncPeriod period, stri protected async Task SyncBlobs(SyncPeriod period, string topic, Guid jobId, CancellationToken cancellationToken, params string[] paths) { var job = syncJobStore.GetJob(jobId); + var degreeOfParallelism = options.Value.GetConcurrency(BusinessOptions.Feature.BlobItems); + job?.Start(); - logger.LogInformation("SyncNotifications period: {Period}, maxDegreeOfParallelism={MaxDegreeOfParallelism}, Environment.ProcessorCount={ProcessorCount}", period.ToString(), maxDegreeOfParallelism, Environment.ProcessorCount); + logger.LogInformation("SyncNotifications period: {Period}, maxDegreeOfParallelism={degreeOfParallelism}, Environment.ProcessorCount={ProcessorCount}", period.ToString(), degreeOfParallelism, Environment.ProcessorCount); try { foreach (var path in paths) diff --git a/Btms.Business/Commands/SyncNotificationsCommand.cs b/Btms.Business/Commands/SyncNotificationsCommand.cs index 5f2ef1f..179aeda 100644 --- a/Btms.Business/Commands/SyncNotificationsCommand.cs +++ b/Btms.Business/Commands/SyncNotificationsCommand.cs @@ -27,12 +27,12 @@ internal class Handler( IOptions businessOptions, ISyncJobStore syncJobStore) : SyncCommand.Handler(syncMetrics, bus, logger, sensitiveDataSerializer, - blobService, syncJobStore) + blobService, businessOptions, syncJobStore) { public override async Task Handle(SyncNotificationsCommand request, CancellationToken cancellationToken) { var rootFolder = string.IsNullOrEmpty(request.RootFolder) - ? businessOptions.Value.DmpBlobRootFolder + ? Options.DmpBlobRootFolder : request.RootFolder; if (request.BlobFiles.Any()) diff --git a/Btms.Consumers/ConsumerOptions.cs b/Btms.Consumers/ConsumerOptions.cs new file mode 100644 index 0000000..8ea4430 --- /dev/null +++ b/Btms.Consumers/ConsumerOptions.cs @@ -0,0 +1,15 @@ +using System.ComponentModel.DataAnnotations; +using Btms.Azure; + +namespace Btms.Consumers; + +public class ConsumerOptions +{ + public const string SectionName = nameof(ConsumerOptions); + + public int InMemoryNotifications { get; set; } = 2; + public int InMemoryGmrs { get; set; } = 2; + public int InMemoryClearanceRequests { get; set; } = 2; + public int InMemoryDecisions { get; set; } = 2; + +} \ No newline at end of file diff --git a/Btms.Consumers/Extensions/ServiceCollectionExtensions.cs b/Btms.Consumers/Extensions/ServiceCollectionExtensions.cs index 04ce9a4..5078f2e 100644 --- a/Btms.Consumers/Extensions/ServiceCollectionExtensions.cs +++ b/Btms.Consumers/Extensions/ServiceCollectionExtensions.cs @@ -1,3 +1,5 @@ +using System.Configuration; +using Btms.Common.Extensions; using Btms.Consumers.Interceptors; using Btms.Consumers.MemoryQueue; using Btms.Metrics.Extensions; @@ -5,6 +7,7 @@ using Btms.Types.Ipaffs; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; using SlimMessageBus.Host; using SlimMessageBus.Host.Interceptor; using SlimMessageBus.Host.Memory; @@ -17,6 +20,17 @@ public static class ServiceCollectionExtensions public static IServiceCollection AddConsumers(this IServiceCollection services, IConfiguration configuration) { + services.BtmsAddOptions(configuration, ConsumerOptions.SectionName); + + var consumerOpts = configuration + .GetSection(ConsumerOptions.SectionName) + .Get() ?? new ConsumerOptions(); + + // services.BtmsAddOptions(configuration, ConsumerOptions.SectionName); + // + // var consumerOpts = services.GetRequiredService>(); + + services.AddBtmsMetrics(); services.AddSingleton(); services.AddSingleton(typeof(IConsumerInterceptor<>), typeof(MetricsInterceptor<>)); @@ -42,24 +56,24 @@ public static IServiceCollection AddConsumers(this IServiceCollection services, .Produce(x => x.DefaultTopic("NOTIFICATIONS")) .Consume(x => { - x.Instances(2); + x.Instances(consumerOpts.InMemoryNotifications); x.Topic("NOTIFICATIONS").WithConsumer(); }) .Produce(x => x.DefaultTopic("GMR")) .Consume(x => { - x.Instances(2); + x.Instances(consumerOpts.InMemoryGmrs); x.Topic("GMR").WithConsumer(); }) .Produce(x => x.DefaultTopic(nameof(AlvsClearanceRequest))) .Consume(x => { - x.Instances(2); - x.Topic("ALVS").WithConsumer(); + x.Instances(consumerOpts.InMemoryClearanceRequests); + x.Topic("CLEARANCEREQUESTS").WithConsumer(); }) .Consume(x => { - x.Instances(2); + x.Instances(consumerOpts.InMemoryDecisions); x.Topic("DECISIONS").WithConsumer(); }); });