From d9c114fc07d3637171a9124e117d36b232ca0fe2 Mon Sep 17 00:00:00 2001 From: Thomas Anderson Date: Wed, 4 Dec 2024 08:01:31 +0000 Subject: [PATCH 1/3] Removed the dependency on Json.Net and stopped flatting and unflatting the Json Now it just finds the value using JsonPath and generates a JsonPatch to update the value with a redacted value --- Btms.SensitiveData/Btms.SensitiveData.csproj | 14 +++-- Btms.SensitiveData/SensitiveDataSerializer.cs | 58 +++++++++++-------- 2 files changed, 41 insertions(+), 31 deletions(-) diff --git a/Btms.SensitiveData/Btms.SensitiveData.csproj b/Btms.SensitiveData/Btms.SensitiveData.csproj index 87e08bb..ec274ad 100644 --- a/Btms.SensitiveData/Btms.SensitiveData.csproj +++ b/Btms.SensitiveData/Btms.SensitiveData.csproj @@ -8,20 +8,22 @@ - + + + + + + + + - - - - - diff --git a/Btms.SensitiveData/SensitiveDataSerializer.cs b/Btms.SensitiveData/SensitiveDataSerializer.cs index 0dc3cd1..4189d11 100644 --- a/Btms.SensitiveData/SensitiveDataSerializer.cs +++ b/Btms.SensitiveData/SensitiveDataSerializer.cs @@ -1,11 +1,12 @@ using System.Text.Json; +using System.Text.Json.Nodes; using System.Text.Json.Serialization; -using System.Text.RegularExpressions; -using JsonFlatten; +using Btms.Common.Extensions; +using Json.Patch; +using Json.Path; +using Json.Pointer; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; -using Newtonsoft.Json.Linq; -using Newtonsoft.Json.Serialization; namespace Btms.SensitiveData; @@ -45,7 +46,7 @@ public T Deserialize(string json, Action optionsOverri } } - + public string RedactRawJson(string json, Type type) { if (options.Value.Include) @@ -53,35 +54,42 @@ public string RedactRawJson(string json, Type type) return json; } var sensitiveFields = SensitiveFieldsProvider.Get(type); - - var jObject = JObject.Parse(json); - var fields = jObject.Flatten(); - - foreach (var field in sensitiveFields) + var rootNode = JsonNode.Parse(json); + + foreach (var sensitiveField in sensitiveFields) { - if (fields.TryGetValue(field, out var value)) + var jsonPath = JsonPath.Parse($"$.{sensitiveField}"); + var result = jsonPath.Evaluate(rootNode); + + foreach (var match in result.Matches) { - if (!options.Value.Include) + JsonPatch patch; + if (match.Value is JsonArray jsonArray) { - fields[field] = options.Value.Getter(value.ToString()!); + var redactedList = jsonArray.Select(x => + { + var redactedValue = options.Value.Getter(x?.GetValue()!); + return redactedValue; + }).ToJson(); + + patch = new JsonPatch(PatchOperation.Replace(JsonPointer.Parse($"{match.Location!.AsJsonPointer()}"), JsonNode.Parse(redactedList))); } - } - else - { - for (int i = 0; i < fields.Keys.Count; i++) + else { - var key = fields.Keys.ElementAt(i); - var replaced = Regex.Replace(key, "\\[.*?\\]", "", RegexOptions.NonBacktracking); - if (replaced == field && fields.TryGetValue(key, out var v) && !options.Value.Include) - { - fields[key] = options.Value.Getter(v.ToString()!); - } + var redactedValue = options.Value.Getter(match.Value?.GetValue()!); + patch = new JsonPatch(PatchOperation.Replace(JsonPointer.Parse(match.Location!.AsJsonPointer()), redactedValue)); + } + + + var patchResult = patch.Apply(rootNode); + if (patchResult.IsSuccess) + { + rootNode = patchResult.Result; } } } - var redactedString = fields.Unflatten().ToString(); - return redactedString; + return rootNode!.ToJsonString(); } } \ No newline at end of file From 8d1e544a0883ae2f0e81015ac0a7cabfc7e3ccfb Mon Sep 17 00:00:00 2001 From: Thomas Anderson Date: Wed, 4 Dec 2024 08:14:11 +0000 Subject: [PATCH 2/3] Updated path on test --- Btms.Backend.IntegrationTests/SensitiveDataTests.cs | 9 +++++++-- Btms.SensitiveData/Btms.SensitiveData.csproj | 6 ------ 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/Btms.Backend.IntegrationTests/SensitiveDataTests.cs b/Btms.Backend.IntegrationTests/SensitiveDataTests.cs index 3174a0c..149fee0 100644 --- a/Btms.Backend.IntegrationTests/SensitiveDataTests.cs +++ b/Btms.Backend.IntegrationTests/SensitiveDataTests.cs @@ -1,4 +1,6 @@ using System.Text.Json.Nodes; +using Btms.Backend.IntegrationTests.Helpers; +using Btms.BlobService; using Btms.SensitiveData; using Btms.Types.Ipaffs; using FluentAssertions; @@ -8,13 +10,15 @@ namespace Btms.Backend.IntegrationTests; + public class SensitiveDataTests { [Fact] public void WhenIncludeSensitiveData_RedactedShouldBeSameAsJson() { + var filePath = "../../../Fixtures/SmokeTest/IPAFFS/CHEDA/CHEDA_GB_2024_1041389-ee0e6fcf-52a4-45ea-8830-d4553ee70361.json"; string json = - File.ReadAllText(Path.GetFullPath("..\\..\\..\\Fixtures\\SmokeTest\\IPAFFS\\CHEDA\\CHEDA_GB_2024_1041389-ee0e6fcf-52a4-45ea-8830-d4553ee70361.json")); + File.ReadAllText(filePath); SensitiveDataOptions options = new SensitiveDataOptions { Getter = s => "TestRedacted", Include = true }; var serializer = new SensitiveDataSerializer(Options.Create(options), NullLogger.Instance); @@ -28,8 +32,9 @@ public void WhenIncludeSensitiveData_RedactedShouldBeSameAsJson() [Fact] public void WhenIncludeSensitiveData_RedactedShouldBeDifferentJson() { + var filePath = "../../../Fixtures/SmokeTest/IPAFFS/CHEDA/CHEDA_GB_2024_1041389-ee0e6fcf-52a4-45ea-8830-d4553ee70361.json"; string json = - File.ReadAllText(Path.GetFullPath("..\\..\\..\\Fixtures\\SmokeTest\\IPAFFS\\CHEDA\\CHEDA_GB_2024_1041389-ee0e6fcf-52a4-45ea-8830-d4553ee70361.json")); + File.ReadAllText(filePath); SensitiveDataOptions options = new SensitiveDataOptions { Getter = s => "TestRedacted", Include = false }; var serializer = new SensitiveDataSerializer(Options.Create(options), NullLogger.Instance); diff --git a/Btms.SensitiveData/Btms.SensitiveData.csproj b/Btms.SensitiveData/Btms.SensitiveData.csproj index ec274ad..da9bad4 100644 --- a/Btms.SensitiveData/Btms.SensitiveData.csproj +++ b/Btms.SensitiveData/Btms.SensitiveData.csproj @@ -7,12 +7,6 @@ true - - - - - - From 131ca6df02561bfb2bdd3d274ef46993cb495a08 Mon Sep 17 00:00:00 2001 From: Thomas Anderson Date: Wed, 4 Dec 2024 08:15:54 +0000 Subject: [PATCH 3/3] Added call to start job --- Btms.Backend/Mediatr/BtmsMediator.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/Btms.Backend/Mediatr/BtmsMediator.cs b/Btms.Backend/Mediatr/BtmsMediator.cs index 9da1baa..9b21c51 100644 --- a/Btms.Backend/Mediatr/BtmsMediator.cs +++ b/Btms.Backend/Mediatr/BtmsMediator.cs @@ -35,6 +35,7 @@ public async Task SendJob(TRequest request, CancellationToken cancella await backgroundTaskQueue.QueueBackgroundWorkItemAsync(async (ct) => { + job.Start(); using var scope = serviceScopeFactory.CreateScope(); using var activity = ActivitySource.StartActivity(ActivityName, ActivityKind.Client); var m = scope.ServiceProvider.GetRequiredService();