Merge remote-tracking branch 'origin/feature/CDMS-144-add-automated-t…

…est' into feature/CDMS-144-add-automated-test

Btms.Analytics.Tests/ImportNotificationsByCommoditiesTests.cs

@@ -5,13 +5,14 @@
 using Xunit.Abstractions;
 using Btms.Analytics.Tests.Helpers;
 using Btms.Analytics.Tests.Fixtures;
+using Btms.Model.Extensions;
 using TestDataGenerator.Config;
 using TestGenerator.IntegrationTesting.Backend;
 
 namespace Btms.Analytics.Tests;
 
 public class ImportNotificationsByCommoditiesTests(ITestOutputHelper output)
-    : ScenarioDatasetBaseTest(output, Datasets.FunctionalAnalyticsDatasetName)
+    : ScenarioDatasetBaseTest(output, Datasets.FunctionalAnalyticsDatasetName, reloadData:false)
 {
 
     [Fact]
@@ -24,6 +25,7 @@ public async Task WhenCalledLastWeek_ReturnExpectedAggregation()
             .ToList();
 
         TestOutputHelper.WriteLine("{0} aggregated items found", result.Count);
+        TestOutputHelper.WriteLine(result.ToJsonString());
 
         result.Count().Should().Be(8);
         result.Select(r => r.Name).Order().Should().Equal(AnalyticsHelpers.GetImportNotificationSegments().Order());

Btms.Analytics.Tests/ImportNotificationsByCreatedDateTests.cs

@@ -32,6 +32,9 @@ public async Task WhenCalledLast48Hours_ReturnExpectedAggregation()
         result[0].Name.Should().Be("CHEDA Linked");
         result[0].Periods[0].Period.Should().BeOnOrBefore(DateTime.Today.Tomorrow());
         result[0].Periods.Count.Should().Be(48);
+
+        result.Sum(r => r.Periods.Sum(p => p.Value))
+            .Should().BeGreaterThan(0);
     }
 
     [Fact]
@@ -58,7 +61,7 @@ public async Task WhenCalledLastMonth_ReturnExpectedAggregation()
             r.Periods.Count.Should().Be(DateTime.Today.DaysSinceMonthAgo() + 1);
         });
 
-        result[0].Periods.Sum(p => p.Value)
+        result.Sum(r => r.Periods.Sum(p => p.Value))
             .Should().BeGreaterThan(0);
     }
 
@@ -90,7 +93,7 @@ public async Task WhenCalledWithTimePeriodYieldingNoResults_ReturnEmptyAggregati
             r.Periods.Count.Should().Be(24);
         });
 
-        result[0].Periods.Sum(p => p.Value)
+        result.Sum(r => r.Periods.Sum(p => p.Value))
             .Should().Be(0);
     }
 }

Btms.Analytics/Extensions/AnalyticsExtensions.cs

@@ -40,6 +40,7 @@ public static string MetricsKey(this DatetimeSeries ds)
 
     public static Dictionary<string, BsonDocument> GetAggregatedRecordsDictionary<T>(
         this IMongoCollectionSet<T> collection,
+        ILogger logger,
         FilterDefinition<T> filter,
         ProjectionDefinition<T> projection,
         ProjectionDefinition<BsonDocument> group,
@@ -52,7 +53,7 @@ public static Dictionary<string, BsonDocument> GetAggregatedRecordsDictionary<T>
             .Project(projection)
             .Group(group)
             .Group(datasetGroup)
-            .ToList()
+            .Execute(logger)
             .ToDictionary(createDatasetName, b => b);
     }
 
@@ -157,7 +158,7 @@ internal static IEnumerable<TSource> Execute<TSource>(this IAggregateFluent<TSou
     /// <returns></returns>
     /// <exception cref="AnalyticsException"></exception>
     public static Dictionary<TKey, TElement> ExecuteAsDictionary<TSource, TKey, TElement>(
-        this IEnumerable<TSource> source,
+        this IQueryable<TSource> source,
         ILogger logger,
         Func<TSource, TKey> keySelector,
         Func<TSource, TElement> elementSelector)

Btms.Analytics/Extensions/ImportNotificationExtensions.cs

@@ -0,0 +1,21 @@
+using Btms.Model;
+using Btms.Model.Ipaffs;
+
+namespace Btms.Analytics.Extensions;
+
+public static class ImportNotificationExtensions
+{   
+    public static IQueryable<ImportNotification> WhereFilteredByCreatedDateAndParams(this IQueryable<ImportNotification> source, DateTime from, DateTime to,
+        ImportNotificationTypeEnum[]? chedTypes = null, string? country = null)
+    {
+        return source
+            .Where(n => (n.CreatedSource >= from && n.CreatedSource < to)
+                && (country == null || n.CommoditiesSummary!.CountryOfOrigin! == country)
+                && (
+                    chedTypes == null ||
+                    chedTypes!.Length == 0 || n.ImportNotificationType == null ||
+                    chedTypes!.Contains(n.ImportNotificationType!.Value)
+                )
+            );
+    }
+}

Btms.Analytics/ImportNotificationsAggregationService.cs

@@ -119,10 +119,7 @@ public Task<SingleSeriesDataset> ByMaxVersion(DateTime from, DateTime to, Import
     {
         var data = context
             .Notifications
-            .Where(n => (n.CreatedSource >= from && n.CreatedSource < to)
-                        && (country == null || n.CommoditiesSummary!.CountryOfOrigin! == country)
-                        && (chedTypes == null || chedTypes!.Length == 0 || n.ImportNotificationType == null || chedTypes!.Contains(n.ImportNotificationType!.Value))
-            )
+            .WhereFilteredByCreatedDateAndParams(from, to, chedTypes, country)
             .GroupBy(n => new { MaxVersion =
                 n.AuditEntries.Where(a => a.CreatedBy == "Ipaffs").Max(a => a.Version )
             })
@@ -149,7 +146,7 @@ private Task<MultiSeriesDatetimeDataset> AggregateByLinkedAndNotificationType(Da
 
         var mongoResult = context
             .Notifications
-            .GetAggregatedRecordsDictionary(filter, projection, group, datasetGroup, createDatasetName);
+            .GetAggregatedRecordsDictionary(logger, filter, projection, group, datasetGroup, createDatasetName);
 
         var output = AnalyticsHelpers.GetImportNotificationSegments()
             .Select(title => mongoResult.AsDataset(dateRange, title))

Btms.Analytics/MovementExceptions.cs

@@ -32,11 +32,12 @@ public class MovementExceptions(IMongoDbContext context, ILogger logger)
                 ItemCount = m.Items.Count,
                 ChedTypes = m.BtmsStatus.ChedTypes,
                 Status = m.BtmsStatus,
-                DecisionMatched = !m.AlvsDecisionStatus.Decisions
-                    .OrderBy(d => d.Context.AlvsDecisionNumber)
-                    .Reverse()
-                    .First()
-                    .Context.DecisionMatched,
+                DecisionMatched = m.AlvsDecisionStatus.Context.DecisionComparison!.DecisionMatched,
+                // DecisionMatched = !m.AlvsDecisionStatus.Decisions
+                //     .OrderBy(d => d.Context.AlvsDecisionNumber)
+                //     .Reverse()
+                //     .First()
+                //     .Context.DecisionMatched,
                 HasNotificationRelationships = m.Relationships.Notifications.Data.Count > 0,
                 ContiguousAlvsClearanceRequestVersionsFrom1 = 
                     m.ClearanceRequests.Select(c => c.Header!.EntryVersionNumber)

Btms.Analytics/MovementsAggregationService.cs

@@ -261,7 +261,7 @@ private Task<MultiSeriesDatetimeDataset> Aggregate(DateTime[] dateRange, Func<Bs
 
         var mongoResult = context
             .Movements
-            .GetAggregatedRecordsDictionary(filter, projection, group, datasetGroup, createDatasetName);
+            .GetAggregatedRecordsDictionary(logger, filter, projection, group, datasetGroup, createDatasetName);
 
         var output = AnalyticsHelpers.GetMovementStatusSegments()
             .Select(title => mongoResult.AsDataset(dateRange, title))
@@ -286,10 +286,11 @@ public Task<SummarisedDataset<SingleSeriesDataset, StringBucketDimensionResult>>
             .WhereFilteredByCreatedDateAndParams(from, to, chedTypes, country)
             .SelectMany(m => m.AlvsDecisionStatus.Decisions.Select(
                 d => new {Decision = d, Movement = m } ))
-            .SelectMany(d => d.Decision.Context.Checks.Select(c => new { d.Decision, d.Movement, Check = c}))
+            .SelectMany(d => d.Decision.Context.DecisionComparison!.Checks.Select(c => new { d.Decision, d.Movement, Check = c}))
+            // .SelectMany(d => d.Decision.Context.Checks.Select(c => new { d.Decision, d.Movement, Check = c}))
             .GroupBy(d => new
             {
-                d.Decision.Context.DecisionStatus,
+                d.Decision.Context.DecisionComparison!.DecisionStatus,
                 d.Check.CheckCode,
                 d.Check.AlvsDecisionCode,
                 d.Check.BtmsDecisionCode

Btms.Backend.Data/Mongo/MongoIndexService.cs

@@ -12,10 +12,20 @@ public Task StartAsync(CancellationToken cancellationToken)
     {
         return Task.WhenAll(
             CreateIndex("MatchReferenceIdx",
-                Builders<ImportNotification>.IndexKeys.Ascending(m => m._MatchReference), cancellationToken),
+                Builders<ImportNotification>.IndexKeys.Ascending(n => n._MatchReference), cancellationToken),
+            CreateIndex("Created",
+                Builders<ImportNotification>.IndexKeys.Ascending(n => n.Created), cancellationToken),
+            CreateIndex("CreatedSource",
+                Builders<ImportNotification>.IndexKeys.Ascending(n => n.CreatedSource), cancellationToken),
 
             CreateIndex("MatchReferenceIdx",
-                Builders<Movement>.IndexKeys.Ascending(m => m._MatchReferences), cancellationToken));
+                Builders<Movement>.IndexKeys.Ascending(m => m._MatchReferences), cancellationToken),
+            CreateIndex("Created",
+                Builders<Movement>.IndexKeys.Ascending(m => m.Created), cancellationToken),
+            CreateIndex("CreatedSource",
+                Builders<Movement>.IndexKeys.Ascending(m => m.CreatedSource), cancellationToken)
+
+        );
 
     }
 

Btms.Backend.IntegrationTests/DecisionTests/AlvsDecisionNumber1Missing.cs

@@ -4,6 +4,7 @@
 using FluentAssertions;
 using TestDataGenerator.Scenarios;
 using TestGenerator.IntegrationTesting.Backend;
+using TestGenerator.IntegrationTesting.Backend.Extensions;
 using TestGenerator.IntegrationTesting.Backend.Fixtures;
 using Xunit;
 using Xunit.Abstractions;
@@ -16,15 +17,32 @@ public class AlvsDecisionNumber1Missing(ITestOutputHelper output)
 {
 
     [Fact]
-    public void ShouldHave2AlvsDecision()
+    public void AlvsDecisionShouldHaveCorrectChecks()
     {
-        // Assert
-        var movement = Client.AsJsonApiClient()
-            .Get("api/movements")
-            .GetResourceObjects<Movement>()
-            .Single();
-
-        movement
+        Client
+            .GetSingleMovement()
+            .AlvsDecisionStatus.Context.DecisionComparison!.Checks
+            .Should().BeEquivalentTo([
+                new { 
+                    ItemNumber = 1,
+                    CheckCode = "H222",
+                    AlvsDecisionCode = "H01", 
+                    BtmsDecisionCode = "X00"
+                },
+                new {
+                    ItemNumber = 1,
+                    CheckCode = "H224",
+                    AlvsDecisionCode = "H01", 
+                    BtmsDecisionCode = "X00"
+                }
+            ]);
+    }
+
+    [Fact]
+    public void ShouldHave1AlvsDecision()
+    {
+        Client
+            .GetSingleMovement()
             .AlvsDecisionStatus
             .Decisions
             .Count
@@ -35,13 +53,8 @@ public void ShouldHave2AlvsDecision()
     [Fact]
     public void ShouldHaveCorrectDecisionNumbers()
     {
-        // Assert
-        var movement = Client.AsJsonApiClient()
-            .Get("api/movements")
-            .GetResourceObjects<Movement>()
-            .Single();
-
-        movement
+        Client
+            .GetSingleMovement()
             .AlvsDecisionStatus
             .Decisions
             .Select(d => d.Context.AlvsDecisionNumber)
@@ -52,17 +65,12 @@ public void ShouldHaveCorrectDecisionNumbers()
     [Fact]
     public void ShouldHaveVersionNotCompleteDecisionStatus()
     {
-
-        // Assert
-        var movement = Client.AsJsonApiClient()
-            .Get("api/movements")
-            .GetResourceObjects<Movement>()
-            .Single();
-
-        movement
+        Client
+            .GetSingleMovement()
             .AlvsDecisionStatus
+            .Context.DecisionComparison!
             .DecisionStatus
             .Should()
-            .Be(DecisionStatusEnum.AlvsDecisionVersion1NotPresent);
+            .Be(DecisionStatusEnum.NoImportNotificationsLinked);
     }
 }

Btms.Backend.IntegrationTests/DecisionTests/ChedPDuplicateDecisionTests.cs

@@ -43,7 +43,7 @@ public void SimpleChedPScenario_ShouldBeLinkedAndMatchDecision()
 
             movement.AlvsDecisionStatus.Decisions
                 .First()
-                .Context.DecisionMatched
+                .Context.DecisionComparison!.DecisionMatched
                 .Should().BeTrue();
 
             var decisionWithLinkAndContext = movement.AuditEntries