Merge pull request #1 from DEFRA/feature/cdms-165

Removed the dependency on Json.Net and stopped flatting and unflattin…
DEFRA · Dec 4, 2024 · 9aa001c · 9aa001c
2 parents 245435a + 131ca6d
commit 9aa001c
Show file tree

Hide file tree

Showing 4 changed files with 43 additions and 33 deletions.
diff --git a/Btms.Backend.IntegrationTests/SensitiveDataTests.cs b/Btms.Backend.IntegrationTests/SensitiveDataTests.cs
@@ -1,4 +1,6 @@
 using System.Text.Json.Nodes;
+using Btms.Backend.IntegrationTests.Helpers;
+using Btms.BlobService;
 using Btms.SensitiveData;
 using Btms.Types.Ipaffs;
 using FluentAssertions;
@@ -8,13 +10,15 @@
 
 namespace Btms.Backend.IntegrationTests;
 
+
 public class SensitiveDataTests
 {
     [Fact]
     public void WhenIncludeSensitiveData_RedactedShouldBeSameAsJson()
     {
+        var filePath = "../../../Fixtures/SmokeTest/IPAFFS/CHEDA/CHEDA_GB_2024_1041389-ee0e6fcf-52a4-45ea-8830-d4553ee70361.json";
         string json =
-            File.ReadAllText(Path.GetFullPath("..\\..\\..\\Fixtures\\SmokeTest\\IPAFFS\\CHEDA\\CHEDA_GB_2024_1041389-ee0e6fcf-52a4-45ea-8830-d4553ee70361.json"));
+            File.ReadAllText(filePath);
 
         SensitiveDataOptions options = new SensitiveDataOptions { Getter = s => "TestRedacted", Include = true };
         var serializer = new SensitiveDataSerializer(Options.Create(options), NullLogger<SensitiveDataSerializer>.Instance);
@@ -28,8 +32,9 @@ public void WhenIncludeSensitiveData_RedactedShouldBeSameAsJson()
     [Fact]
     public void WhenIncludeSensitiveData_RedactedShouldBeDifferentJson()
     {
+        var filePath = "../../../Fixtures/SmokeTest/IPAFFS/CHEDA/CHEDA_GB_2024_1041389-ee0e6fcf-52a4-45ea-8830-d4553ee70361.json";
         string json =
-            File.ReadAllText(Path.GetFullPath("..\\..\\..\\Fixtures\\SmokeTest\\IPAFFS\\CHEDA\\CHEDA_GB_2024_1041389-ee0e6fcf-52a4-45ea-8830-d4553ee70361.json"));
+            File.ReadAllText(filePath);
 
         SensitiveDataOptions options = new SensitiveDataOptions { Getter = s => "TestRedacted", Include = false };
         var serializer = new SensitiveDataSerializer(Options.Create(options), NullLogger<SensitiveDataSerializer>.Instance);

diff --git a/Btms.Backend/Mediatr/BtmsMediator.cs b/Btms.Backend/Mediatr/BtmsMediator.cs
@@ -35,6 +35,7 @@ public async Task SendJob<TRequest>(TRequest request, CancellationToken cancella
 
             await backgroundTaskQueue.QueueBackgroundWorkItemAsync(async (ct) =>
             {
+                job.Start();
                 using var scope = serviceScopeFactory.CreateScope();
                 using var activity = ActivitySource.StartActivity(ActivityName, ActivityKind.Client);
                 var m = scope.ServiceProvider.GetRequiredService<IMediator>();

diff --git a/Btms.SensitiveData/Btms.SensitiveData.csproj b/Btms.SensitiveData/Btms.SensitiveData.csproj
@@ -8,20 +8,16 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="JsonFlatten" Version="1.0.4" />
+    <PackageReference Include="JsonPatch.Net" Version="3.2.2" />
+    <PackageReference Include="JsonPath.Net" Version="2.0.0" />
     <PackageReference Include="Microsoft.Extensions.Configuration" Version="8.0.0" />
     <PackageReference Include="Microsoft.Extensions.Configuration.Abstractions" Version="8.0.0" />
     <PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="8.0.2" />
     <PackageReference Include="Microsoft.Extensions.Options" Version="8.0.0" />
-    <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
   </ItemGroup>
 
   <ItemGroup>
     <ProjectReference Include="..\Btms.Common\Btms.Common.csproj" />
   </ItemGroup>
 
-  <ItemGroup>
-    <Folder Include="Flattener\" />
-  </ItemGroup>
-
 </Project>
diff --git a/Btms.SensitiveData/SensitiveDataSerializer.cs b/Btms.SensitiveData/SensitiveDataSerializer.cs
@@ -1,11 +1,12 @@
 using System.Text.Json;
+using System.Text.Json.Nodes;
 using System.Text.Json.Serialization;
-using System.Text.RegularExpressions;
-using JsonFlatten;
+using Btms.Common.Extensions;
+using Json.Patch;
+using Json.Path;
+using Json.Pointer;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
-using Newtonsoft.Json.Linq;
-using Newtonsoft.Json.Serialization;
 
 namespace Btms.SensitiveData;
 
@@ -45,43 +46,50 @@ public T Deserialize<T>(string json, Action<JsonSerializerOptions> optionsOverri
         }
 
     }
-
+   
     public string RedactRawJson(string json, Type type)
     {
         if (options.Value.Include)
         {
             return json;
         }
         var sensitiveFields = SensitiveFieldsProvider.Get(type);
-
-        var jObject = JObject.Parse(json);
 
-        var fields = jObject.Flatten();
-
-        foreach (var field in sensitiveFields)
+        var rootNode = JsonNode.Parse(json);
+       
+        foreach (var sensitiveField in sensitiveFields)
         {
-            if (fields.TryGetValue(field, out var value))
+            var jsonPath = JsonPath.Parse($"$.{sensitiveField}");
+            var result = jsonPath.Evaluate(rootNode);
+
+            foreach (var match in result.Matches)
             {
-                if (!options.Value.Include)
+                JsonPatch patch;
+                if (match.Value is JsonArray jsonArray)
                 {
-                    fields[field] = options.Value.Getter(value.ToString()!);
+                    var redactedList = jsonArray.Select(x =>
+                    {
+                        var redactedValue = options.Value.Getter(x?.GetValue<string>()!);
+                        return redactedValue;
+                    }).ToJson();
+
+                    patch = new JsonPatch(PatchOperation.Replace(JsonPointer.Parse($"{match.Location!.AsJsonPointer()}"), JsonNode.Parse(redactedList)));
                 }
-            }
-            else
-            {
-                for (int i = 0; i < fields.Keys.Count; i++)
+                else
                 {
-                    var key = fields.Keys.ElementAt(i);
-                    var replaced = Regex.Replace(key, "\\[.*?\\]", "", RegexOptions.NonBacktracking);
-                    if (replaced == field && fields.TryGetValue(key, out var v) && !options.Value.Include)
-                    {
-                        fields[key] = options.Value.Getter(v.ToString()!);
-                    }
+                    var redactedValue = options.Value.Getter(match.Value?.GetValue<string>()!);
+                    patch = new JsonPatch(PatchOperation.Replace(JsonPointer.Parse(match.Location!.AsJsonPointer()), redactedValue));
+                }
+
+
+                var patchResult = patch.Apply(rootNode);
+                if (patchResult.IsSuccess)
+                {
+                    rootNode = patchResult.Result;
                 }
             }
         }
 
-        var redactedString = fields.Unflatten().ToString();
-        return redactedString;
+        return rootNode!.ToJsonString();
     }
 }