From 462b4db8b02d0428565f9285f9f0943ea975f4ea Mon Sep 17 00:00:00 2001 From: Thomas Anderson Date: Tue, 3 Dec 2024 10:47:22 +0000 Subject: [PATCH 1/5] Added the ability to redact data without Deserializing to an object. This will still look at the [SensitiveDataAttribute] so there is no need for additional configuration --- .../SensitiveDataSerializerTests.cs | 31 ++++++++ Cdms.SensitiveData/Cdms.SensitiveData.csproj | 6 ++ .../ISensitiveDataSerializer.cs | 5 +- Cdms.SensitiveData/SensitiveDataSerializer.cs | 35 ++++++++++ Cdms.SensitiveData/SensitiveFieldsProvider.cs | 70 +++++++++++++++++++ 5 files changed, 146 insertions(+), 1 deletion(-) create mode 100644 Cdms.SensitiveData/SensitiveFieldsProvider.cs diff --git a/Cdms.SensitiveData.Tests/SensitiveDataSerializerTests.cs b/Cdms.SensitiveData.Tests/SensitiveDataSerializerTests.cs index b983e54..451bce2 100644 --- a/Cdms.SensitiveData.Tests/SensitiveDataSerializerTests.cs +++ b/Cdms.SensitiveData.Tests/SensitiveDataSerializerTests.cs @@ -67,4 +67,35 @@ public void WhenIncludeSensitiveData_ThenDataShouldNotBeRedacted() result.SimpleStringArrayTwo[0].Should().Be("Test String Array Two Item One"); result.SimpleStringArrayTwo[1].Should().Be("Test String Array Two Item Two"); } + + [Fact] + public void WhenDoNotIncludeSensitiveData_AndRequestForRawJson_ThenDataShouldBeRedacted() + { + // ARRANGE + SensitiveDataOptions options = new SensitiveDataOptions { Getter = s => "TestRedacted", Include = false }; + var serializer = new SensitiveDataSerializer(Options.Create(options), NullLogger.Instance); + + var simpleClass = new SimpleClass() + { + SimpleStringOne = "Test String One", + SimpleStringTwo = "Test String Two", + SimpleStringArrayOne = + new[] { "Test String Array One Item One", "Test String Array One Item Two" }, + SimpleStringArrayTwo = new[] { "Test String Array Two Item One", "Test String Array Two Item Two" } + }; + + var json = JsonSerializer.Serialize(simpleClass, new JsonSerializerOptions() { PropertyNamingPolicy = JsonNamingPolicy.CamelCase}); + + // ACT + var result = serializer.RedactRawJson(json); + + // ASSERT + var resultClass = JsonSerializer.Deserialize(result, new JsonSerializerOptions() { PropertyNameCaseInsensitive = true }); + resultClass?.SimpleStringOne.Should().Be("TestRedacted"); + resultClass?.SimpleStringTwo.Should().Be("Test String Two"); + resultClass?.SimpleStringArrayOne[0].Should().Be("TestRedacted"); + resultClass?.SimpleStringArrayOne[1].Should().Be("TestRedacted"); + resultClass?.SimpleStringArrayTwo[0].Should().Be("Test String Array Two Item One"); + resultClass?.SimpleStringArrayTwo[1].Should().Be("Test String Array Two Item Two"); + } } \ No newline at end of file diff --git a/Cdms.SensitiveData/Cdms.SensitiveData.csproj b/Cdms.SensitiveData/Cdms.SensitiveData.csproj index 8868104..2b9ff2a 100644 --- a/Cdms.SensitiveData/Cdms.SensitiveData.csproj +++ b/Cdms.SensitiveData/Cdms.SensitiveData.csproj @@ -8,14 +8,20 @@ + + + + + + diff --git a/Cdms.SensitiveData/ISensitiveDataSerializer.cs b/Cdms.SensitiveData/ISensitiveDataSerializer.cs index 8056a84..ded26fc 100644 --- a/Cdms.SensitiveData/ISensitiveDataSerializer.cs +++ b/Cdms.SensitiveData/ISensitiveDataSerializer.cs @@ -5,4 +5,7 @@ namespace Cdms.SensitiveData; public interface ISensitiveDataSerializer { public T Deserialize(string json, Action optionsOverride = null!); -} \ No newline at end of file + + string RedactRawJson(string json); +} + diff --git a/Cdms.SensitiveData/SensitiveDataSerializer.cs b/Cdms.SensitiveData/SensitiveDataSerializer.cs index c129ed4..82a4905 100644 --- a/Cdms.SensitiveData/SensitiveDataSerializer.cs +++ b/Cdms.SensitiveData/SensitiveDataSerializer.cs @@ -1,7 +1,11 @@ using System.Text.Json; using System.Text.Json.Serialization; +using System.Text.RegularExpressions; +using JsonFlatten; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; +using Newtonsoft.Json.Linq; +using Newtonsoft.Json.Serialization; namespace Cdms.SensitiveData; @@ -41,4 +45,35 @@ public T Deserialize(string json, Action optionsOverri } } + + public string RedactRawJson(string json) + { + var sensitiveFields = SensitiveFieldsProvider.Get(); + var jObject = JObject.Parse(json); + + var fields = jObject.Flatten(); + + foreach (var field in sensitiveFields) + { + if (fields.TryGetValue(field, out var value)) + { + fields[field] = options.Value.Getter(value.ToString()!); + } + else + { + for (int i = 0; i < fields.Keys.Count; i++) + { + var key = fields.Keys.ElementAt(i); + var replaced = Regex.Replace(key, "\\[.*?\\]", ""); + if (replaced == field && fields.TryGetValue(key, out var v)) + { + fields[key] = options.Value.Getter(v.ToString()!); + } + } + } + } + + var redactedString = fields.Unflatten().ToString(); + return redactedString; + } } \ No newline at end of file diff --git a/Cdms.SensitiveData/SensitiveFieldsProvider.cs b/Cdms.SensitiveData/SensitiveFieldsProvider.cs new file mode 100644 index 0000000..2bc80b3 --- /dev/null +++ b/Cdms.SensitiveData/SensitiveFieldsProvider.cs @@ -0,0 +1,70 @@ +using System.Text.Json; + +namespace Cdms.SensitiveData; + +public static class SensitiveFieldsProvider +{ + private static Dictionary> cache = new(); + private static readonly object cacheLock = new (); + public static List Get() + { + lock (cacheLock) + { + if (cache.TryGetValue(typeof(T), out var value)) + { + return value; + } + + var type = typeof(T); + + var list = GetSensitiveFields(string.Empty, type); + cache.Add(typeof(T), list); + return list; + } + + + } + + private static List GetSensitiveFields(string root, Type type) + { + var namingPolicy = JsonNamingPolicy.CamelCase; + var list = new List(); + foreach (var property in type.GetProperties()) + { + string currentPath; + currentPath = string.IsNullOrEmpty(root) ? $"{namingPolicy.ConvertName(property.Name)}" : $"{namingPolicy.ConvertName(root)}.{namingPolicy.ConvertName(property.Name)}"; + + if (property.CustomAttributes.Any(x => x.AttributeType == typeof(SensitiveDataAttribute))) + { + list.Add(currentPath); + } + else + { + Type elementType = null!; + + //is a complex type of list of complex type + if (property.PropertyType.IsArray) + { + elementType = property.PropertyType.GetElementType()!; + } + else if (property.PropertyType.IsGenericType) + { + elementType = property.PropertyType.GetGenericArguments()[0]; + + } + else if (property.PropertyType.IsClass) + { + elementType = property.PropertyType; + + } + + if (elementType != null && elementType.Namespace != "System") + { + list.AddRange(GetSensitiveFields($"{currentPath}", elementType!)); + } + } + } + + return list; + } +} \ No newline at end of file From 14a51f9c6743fa55c416017200a5445a98bd54e6 Mon Sep 17 00:00:00 2001 From: Thomas Anderson Date: Tue, 3 Dec 2024 11:45:13 +0000 Subject: [PATCH 2/5] Added endpoints for download data --- .../Commands/DownloadNotificationsCommand.cs | 83 +++++++++++++++++++ .../SensitiveDataSerializerTests.cs | 2 +- .../ISensitiveDataSerializer.cs | 2 +- Cdms.SensitiveData/SensitiveDataSerializer.cs | 4 +- Cdms.SensitiveData/SensitiveFieldsProvider.cs | 17 ++++ CdmsBackend/Endpoints/SyncEndpoints.cs | 21 +++++ 6 files changed, 125 insertions(+), 4 deletions(-) create mode 100644 Cdms.Business/Commands/DownloadNotificationsCommand.cs diff --git a/Cdms.Business/Commands/DownloadNotificationsCommand.cs b/Cdms.Business/Commands/DownloadNotificationsCommand.cs new file mode 100644 index 0000000..86eaecb --- /dev/null +++ b/Cdms.Business/Commands/DownloadNotificationsCommand.cs @@ -0,0 +1,83 @@ +using System.Dynamic; +using System.IO; +using System.IO.Compression; +using System.Text.Json.Serialization; +using Bogus; +using Cdms.BlobService; +using System.Threading; +using MediatR; +using Newtonsoft.Json; +using JsonSerializer = System.Text.Json.JsonSerializer; +using SharpCompress.Writers; +using Cdms.SensitiveData; +using Cdms.Types.Ipaffs; +using Microsoft.AspNetCore.Hosting; + +namespace Cdms.Business.Commands; + +public class DownloadCommand : IRequest +{ + [System.Text.Json.Serialization.JsonConverter(typeof(JsonStringEnumConverter))] + public SyncPeriod SyncPeriod { get; set; } + + public string Path { get; set; } = null!; + + public Type Type { get; set; } = null!; + + internal class Handler(IBlobService blobService, ISensitiveDataSerializer sensitiveDataSerializer, IWebHostEnvironment env) : IRequestHandler + { + + public async Task Handle(DownloadCommand request, CancellationToken cancellationToken) + { + string subFolder = $"{request.Type.Name}\\{Guid.NewGuid()}"; + string rootFolder = System.IO.Path.Combine(env.ContentRootPath, subFolder); + Directory.CreateDirectory(rootFolder); + ParallelOptions options = new() { CancellationToken = cancellationToken, MaxDegreeOfParallelism = 10 }; + var result = blobService.GetResourcesAsync($"{request.Path}{GetPeriodPath(request.SyncPeriod)}", cancellationToken); + + //Write local files + await Parallel.ForEachAsync(result, options, async (item, token) => + { + var blobContent = await blobService.GetResource(item, cancellationToken); + string redactedContent = sensitiveDataSerializer.RedactRawJson(blobContent, request.Type); + var filename = System.IO.Path.Combine(rootFolder, item.Name.Replace('/', System.IO.Path.DirectorySeparatorChar)); + Directory.CreateDirectory(System.IO.Path.GetDirectoryName(filename)!); + await File.WriteAllTextAsync(filename, redactedContent, cancellationToken); + }); + + MemoryStream zipStream = new MemoryStream(); + ZipFile.CreateFromDirectory(rootFolder, zipStream); + zipStream.Position = 0; + var commandResult = new Result(zipStream.ToArray()); + Directory.Delete(rootFolder, true); + return commandResult; + } + + + private static string GetPeriodPath(SyncPeriod period) + { + if (period == SyncPeriod.LastMonth) + { + return DateTime.Today.AddMonths(-1).ToString("/yyyy/MM/"); + } + else if (period == SyncPeriod.ThisMonth) + { + return DateTime.Today.ToString("/yyyy/MM/"); + } + else if (period == SyncPeriod.Today) + { + return DateTime.Today.ToString("/yyyy/MM/dd/"); + } + else if (period == SyncPeriod.All) + { + return "/"; + } + else + { + throw new ArgumentException($"Unexpected SyncPeriod {period}"); + } + } + } + + public record Result(byte[] Zip); +} \ No newline at end of file diff --git a/Cdms.SensitiveData.Tests/SensitiveDataSerializerTests.cs b/Cdms.SensitiveData.Tests/SensitiveDataSerializerTests.cs index 451bce2..c1ec848 100644 --- a/Cdms.SensitiveData.Tests/SensitiveDataSerializerTests.cs +++ b/Cdms.SensitiveData.Tests/SensitiveDataSerializerTests.cs @@ -87,7 +87,7 @@ public void WhenDoNotIncludeSensitiveData_AndRequestForRawJson_ThenDataShouldBeR var json = JsonSerializer.Serialize(simpleClass, new JsonSerializerOptions() { PropertyNamingPolicy = JsonNamingPolicy.CamelCase}); // ACT - var result = serializer.RedactRawJson(json); + var result = serializer.RedactRawJson(json, typeof(SimpleClass)); // ASSERT var resultClass = JsonSerializer.Deserialize(result, new JsonSerializerOptions() { PropertyNameCaseInsensitive = true }); diff --git a/Cdms.SensitiveData/ISensitiveDataSerializer.cs b/Cdms.SensitiveData/ISensitiveDataSerializer.cs index ded26fc..e745d54 100644 --- a/Cdms.SensitiveData/ISensitiveDataSerializer.cs +++ b/Cdms.SensitiveData/ISensitiveDataSerializer.cs @@ -6,6 +6,6 @@ public interface ISensitiveDataSerializer { public T Deserialize(string json, Action optionsOverride = null!); - string RedactRawJson(string json); + string RedactRawJson(string json, Type type); } diff --git a/Cdms.SensitiveData/SensitiveDataSerializer.cs b/Cdms.SensitiveData/SensitiveDataSerializer.cs index 82a4905..ef8c8ad 100644 --- a/Cdms.SensitiveData/SensitiveDataSerializer.cs +++ b/Cdms.SensitiveData/SensitiveDataSerializer.cs @@ -46,9 +46,9 @@ public T Deserialize(string json, Action optionsOverri } - public string RedactRawJson(string json) + public string RedactRawJson(string json, Type type) { - var sensitiveFields = SensitiveFieldsProvider.Get(); + var sensitiveFields = SensitiveFieldsProvider.Get(type); var jObject = JObject.Parse(json); var fields = jObject.Flatten(); diff --git a/Cdms.SensitiveData/SensitiveFieldsProvider.cs b/Cdms.SensitiveData/SensitiveFieldsProvider.cs index 2bc80b3..d76776f 100644 --- a/Cdms.SensitiveData/SensitiveFieldsProvider.cs +++ b/Cdms.SensitiveData/SensitiveFieldsProvider.cs @@ -23,6 +23,23 @@ public static List Get() } + } + + public static List Get(Type type) + { + lock (cacheLock) + { + if (cache.TryGetValue(type, out var value)) + { + return value; + } + + var list = GetSensitiveFields(string.Empty, type); + cache.Add(type, list); + return list; + } + + } private static List GetSensitiveFields(string root, Type type) diff --git a/CdmsBackend/Endpoints/SyncEndpoints.cs b/CdmsBackend/Endpoints/SyncEndpoints.cs index ae09ac1..a79c9a0 100644 --- a/CdmsBackend/Endpoints/SyncEndpoints.cs +++ b/CdmsBackend/Endpoints/SyncEndpoints.cs @@ -1,8 +1,14 @@ +using System.IO.Compression; +using Azure.Storage.Blobs; +using Azure.Storage.Blobs.Models; using Cdms.Business.Commands; using Cdms.Consumers.MemoryQueue; using Cdms.SyncJob; +using Cdms.Types.Alvs; +using Cdms.Types.Ipaffs; using CdmsBackend.Config; using CdmsBackend.Mediatr; +using MediatR; using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Options; @@ -20,6 +26,9 @@ public static void UseSyncEndpoints(this IEndpointRouteBuilder app, IOptions InitialiseEnvironment(IHost app, SyncPeriod return Results.Ok(); } + private static async Task DownloadNotifications([FromServices] IMediator mediator, [FromQuery] string path, [FromQuery]SyncPeriod period) + { + var result = await mediator.Send(new DownloadCommand() { Type = typeof(ImportNotification), Path = path, SyncPeriod = period }); + return Results.File(result.Zip, "application/zip", "notifications.zip"); + } + + private static async Task DownloadClearanceRequests([FromServices] IMediator mediator, [FromQuery] string path, [FromQuery] SyncPeriod period) + { + var result = await mediator.Send(new DownloadCommand() { Type = typeof(AlvsClearanceRequest), Path = path, SyncPeriod = period }); + return Results.File(result.Zip, "application/zip", "clearancerequests.zip"); + } + private static Task GetAllSyncJobs([FromServices] ISyncJobStore store) { return Task.FromResult(Results.Ok(new { jobs = store.GetJobs() })); From 99e2413463a34c14fc3191687fed6c9344508684 Mon Sep 17 00:00:00 2001 From: Thomas Anderson Date: Tue, 3 Dec 2024 11:53:43 +0000 Subject: [PATCH 3/5] fixed some sonar issues --- .../Commands/DownloadNotificationsCommand.cs | 26 +------------ Cdms.Business/Commands/SyncHandler.cs | 28 +------------ .../Commands/SyncPeriodExtensions.cs | 28 +++++++++++++ Cdms.SensitiveData/SensitiveDataSerializer.cs | 2 +- Cdms.SensitiveData/SensitiveFieldsProvider.cs | 39 +++++++++++-------- 5 files changed, 53 insertions(+), 70 deletions(-) create mode 100644 Cdms.Business/Commands/SyncPeriodExtensions.cs diff --git a/Cdms.Business/Commands/DownloadNotificationsCommand.cs b/Cdms.Business/Commands/DownloadNotificationsCommand.cs index 86eaecb..453599b 100644 --- a/Cdms.Business/Commands/DownloadNotificationsCommand.cs +++ b/Cdms.Business/Commands/DownloadNotificationsCommand.cs @@ -33,7 +33,7 @@ public async Task Handle(DownloadCommand request, CancellationToken canc string rootFolder = System.IO.Path.Combine(env.ContentRootPath, subFolder); Directory.CreateDirectory(rootFolder); ParallelOptions options = new() { CancellationToken = cancellationToken, MaxDegreeOfParallelism = 10 }; - var result = blobService.GetResourcesAsync($"{request.Path}{GetPeriodPath(request.SyncPeriod)}", cancellationToken); + var result = blobService.GetResourcesAsync($"{request.Path}{request.SyncPeriod.GetPeriodPath()}", cancellationToken); //Write local files await Parallel.ForEachAsync(result, options, async (item, token) => @@ -53,30 +53,6 @@ await Parallel.ForEachAsync(result, options, async (item, token) => return commandResult; } - - private static string GetPeriodPath(SyncPeriod period) - { - if (period == SyncPeriod.LastMonth) - { - return DateTime.Today.AddMonths(-1).ToString("/yyyy/MM/"); - } - else if (period == SyncPeriod.ThisMonth) - { - return DateTime.Today.ToString("/yyyy/MM/"); - } - else if (period == SyncPeriod.Today) - { - return DateTime.Today.ToString("/yyyy/MM/dd/"); - } - else if (period == SyncPeriod.All) - { - return "/"; - } - else - { - throw new ArgumentException($"Unexpected SyncPeriod {period}"); - } - } } public record Result(byte[] Zip); diff --git a/Cdms.Business/Commands/SyncHandler.cs b/Cdms.Business/Commands/SyncHandler.cs index e374692..667751c 100644 --- a/Cdms.Business/Commands/SyncHandler.cs +++ b/Cdms.Business/Commands/SyncHandler.cs @@ -103,7 +103,7 @@ await Parallel.ForEachAsync(paths, protected async Task SyncBlobPath(string path, SyncPeriod period, string topic, SyncJob.SyncJob job, CancellationToken cancellationToken) { - var result = blobService.GetResourcesAsync($"{path}{GetPeriodPath(period)}", cancellationToken); + var result = blobService.GetResourcesAsync($"{path}{period.GetPeriodPath()}", cancellationToken); await Parallel.ForEachAsync(result, new ParallelOptions() { CancellationToken = cancellationToken, MaxDegreeOfParallelism = maxDegreeOfParallelism }, async (item, token) => { @@ -184,31 +184,5 @@ await bus.Publish(message, } } } - - - - private static string GetPeriodPath(SyncPeriod period) - { - if (period == SyncPeriod.LastMonth) - { - return DateTime.Today.AddMonths(-1).ToString("/yyyy/MM/"); - } - else if (period == SyncPeriod.ThisMonth) - { - return DateTime.Today.ToString("/yyyy/MM/"); - } - else if (period == SyncPeriod.Today) - { - return DateTime.Today.ToString("/yyyy/MM/dd/"); - } - else if (period == SyncPeriod.All) - { - return "/"; - } - else - { - throw new ArgumentException($"Unexpected SyncPeriod {period}"); - } - } } } \ No newline at end of file diff --git a/Cdms.Business/Commands/SyncPeriodExtensions.cs b/Cdms.Business/Commands/SyncPeriodExtensions.cs new file mode 100644 index 0000000..51bb13f --- /dev/null +++ b/Cdms.Business/Commands/SyncPeriodExtensions.cs @@ -0,0 +1,28 @@ +namespace Cdms.Business.Commands; + +public static class SyncPeriodExtensions +{ + public static string GetPeriodPath(this SyncPeriod period) + { + if (period == SyncPeriod.LastMonth) + { + return DateTime.Today.AddMonths(-1).ToString("/yyyy/MM/"); + } + else if (period == SyncPeriod.ThisMonth) + { + return DateTime.Today.ToString("/yyyy/MM/"); + } + else if (period == SyncPeriod.Today) + { + return DateTime.Today.ToString("/yyyy/MM/dd/"); + } + else if (period == SyncPeriod.All) + { + return "/"; + } + else + { + throw new ArgumentException($"Unexpected SyncPeriod {period}"); + } + } +} \ No newline at end of file diff --git a/Cdms.SensitiveData/SensitiveDataSerializer.cs b/Cdms.SensitiveData/SensitiveDataSerializer.cs index ef8c8ad..1a2a8d2 100644 --- a/Cdms.SensitiveData/SensitiveDataSerializer.cs +++ b/Cdms.SensitiveData/SensitiveDataSerializer.cs @@ -64,7 +64,7 @@ public string RedactRawJson(string json, Type type) for (int i = 0; i < fields.Keys.Count; i++) { var key = fields.Keys.ElementAt(i); - var replaced = Regex.Replace(key, "\\[.*?\\]", ""); + var replaced = Regex.Replace(key, "\\[.*?\\]", "", RegexOptions.NonBacktracking); if (replaced == field && fields.TryGetValue(key, out var v)) { fields[key] = options.Value.Getter(v.ToString()!); diff --git a/Cdms.SensitiveData/SensitiveFieldsProvider.cs b/Cdms.SensitiveData/SensitiveFieldsProvider.cs index d76776f..3081b35 100644 --- a/Cdms.SensitiveData/SensitiveFieldsProvider.cs +++ b/Cdms.SensitiveData/SensitiveFieldsProvider.cs @@ -1,3 +1,4 @@ +using System.Reflection; using System.Text.Json; namespace Cdms.SensitiveData; @@ -57,23 +58,7 @@ private static List GetSensitiveFields(string root, Type type) } else { - Type elementType = null!; - - //is a complex type of list of complex type - if (property.PropertyType.IsArray) - { - elementType = property.PropertyType.GetElementType()!; - } - else if (property.PropertyType.IsGenericType) - { - elementType = property.PropertyType.GetGenericArguments()[0]; - - } - else if (property.PropertyType.IsClass) - { - elementType = property.PropertyType; - - } + Type elementType = GetElementType(property)!; if (elementType != null && elementType.Namespace != "System") { @@ -84,4 +69,24 @@ private static List GetSensitiveFields(string root, Type type) return list; } + + private static Type? GetElementType(PropertyInfo property) + { + if (property.PropertyType.IsArray) + { + return property.PropertyType.GetElementType()!; + } + else if (property.PropertyType.IsGenericType) + { + return property.PropertyType.GetGenericArguments()[0]; + + } + else if (property.PropertyType.IsClass) + { + return property.PropertyType; + + } + + return default; + } } \ No newline at end of file From 4ce11e755501c6c20b93383e914da1ec39be4b24 Mon Sep 17 00:00:00 2001 From: Thomas Anderson Date: Tue, 3 Dec 2024 13:59:08 +0000 Subject: [PATCH 4/5] updated to download everything within a single bundle --- .../Commands/DownloadNotificationsCommand.cs | 52 +++++++++++++------ CdmsBackend/Endpoints/SyncEndpoints.cs | 18 ++++--- CdmsBackend/Mediatr/CdmsMediator.cs | 14 +++++ CdmsBackend/Mediatr/ICdmsMediator.cs | 5 +- 4 files changed, 63 insertions(+), 26 deletions(-) diff --git a/Cdms.Business/Commands/DownloadNotificationsCommand.cs b/Cdms.Business/Commands/DownloadNotificationsCommand.cs index 453599b..be241fd 100644 --- a/Cdms.Business/Commands/DownloadNotificationsCommand.cs +++ b/Cdms.Business/Commands/DownloadNotificationsCommand.cs @@ -12,48 +12,66 @@ using Cdms.SensitiveData; using Cdms.Types.Ipaffs; using Microsoft.AspNetCore.Hosting; +using Cdms.SyncJob; +using Cdms.Types.Alvs; +using Cdms.Types.Gvms; namespace Cdms.Business.Commands; -public class DownloadCommand : IRequest +public class DownloadCommand : IRequest, ISyncJob { [System.Text.Json.Serialization.JsonConverter(typeof(JsonStringEnumConverter))] public SyncPeriod SyncPeriod { get; set; } - public string Path { get; set; } = null!; + public Guid JobId { get; } = Guid.NewGuid(); + public string Timespan { get; } = null!; + public string Resource { get; } = null!; - public Type Type { get; set; } = null!; - - internal class Handler(IBlobService blobService, ISensitiveDataSerializer sensitiveDataSerializer, IWebHostEnvironment env) : IRequestHandler + internal class Handler(IBlobService blobService, ISensitiveDataSerializer sensitiveDataSerializer, IWebHostEnvironment env) : IRequestHandler { - public async Task Handle(DownloadCommand request, CancellationToken cancellationToken) + public async Task Handle(DownloadCommand request, CancellationToken cancellationToken) { - string subFolder = $"{request.Type.Name}\\{Guid.NewGuid()}"; - string rootFolder = System.IO.Path.Combine(env.ContentRootPath, subFolder); + string subFolder = $"temp\\{request.JobId}"; + string rootFolder = Path.Combine(env.ContentRootPath, subFolder); Directory.CreateDirectory(rootFolder); + + await Download(request, rootFolder, "RAW/IPAFFS/CHEDA", typeof(ImportNotification), cancellationToken); + await Download(request, rootFolder, "RAW/IPAFFS/CHEDD", typeof(ImportNotification), cancellationToken); + await Download(request, rootFolder, "RAW/IPAFFS/CHEDP", typeof(ImportNotification), cancellationToken); + await Download(request, rootFolder, "RAW/IPAFFS/CHEDPP", typeof(ImportNotification), cancellationToken); + + await Download(request, rootFolder, "RAW/ALVS", typeof(AlvsClearanceRequest), cancellationToken); + + await Download(request, rootFolder, "RAW/GVMSAPIRESPONSE", typeof(SearchGmrsForDeclarationIdsResponse), cancellationToken); + + await Download(request, rootFolder, "RAW/DECISIONS", typeof(AlvsClearanceRequest), cancellationToken); + + ZipFile.CreateFromDirectory(rootFolder, $"{env.ContentRootPath}\\{request.JobId}.zip"); + + Directory.Delete(rootFolder, true); + } + + private async Task Download(DownloadCommand request, string rootFolder, string folder, Type type, CancellationToken cancellationToken) + { + ParallelOptions options = new() { CancellationToken = cancellationToken, MaxDegreeOfParallelism = 10 }; - var result = blobService.GetResourcesAsync($"{request.Path}{request.SyncPeriod.GetPeriodPath()}", cancellationToken); + var result = blobService.GetResourcesAsync($"{folder}{request.SyncPeriod.GetPeriodPath()}", cancellationToken); //Write local files await Parallel.ForEachAsync(result, options, async (item, token) => { var blobContent = await blobService.GetResource(item, cancellationToken); - string redactedContent = sensitiveDataSerializer.RedactRawJson(blobContent, request.Type); + string redactedContent = sensitiveDataSerializer.RedactRawJson(blobContent, type); var filename = System.IO.Path.Combine(rootFolder, item.Name.Replace('/', System.IO.Path.DirectorySeparatorChar)); Directory.CreateDirectory(System.IO.Path.GetDirectoryName(filename)!); await File.WriteAllTextAsync(filename, redactedContent, cancellationToken); }); - - MemoryStream zipStream = new MemoryStream(); - ZipFile.CreateFromDirectory(rootFolder, zipStream); - zipStream.Position = 0; - var commandResult = new Result(zipStream.ToArray()); - Directory.Delete(rootFolder, true); - return commandResult; } } public record Result(byte[] Zip); + + } \ No newline at end of file diff --git a/CdmsBackend/Endpoints/SyncEndpoints.cs b/CdmsBackend/Endpoints/SyncEndpoints.cs index a79c9a0..b5f51b1 100644 --- a/CdmsBackend/Endpoints/SyncEndpoints.cs +++ b/CdmsBackend/Endpoints/SyncEndpoints.cs @@ -11,6 +11,7 @@ using MediatR; using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Options; +using static Cdms.Business.Commands.DownloadCommand; namespace CdmsBackend.Endpoints; @@ -27,8 +28,8 @@ public static void UseSyncEndpoints(this IEndpointRouteBuilder app, IOptions InitialiseEnvironment(IHost app, SyncPeriod return Results.Ok(); } - private static async Task DownloadNotifications([FromServices] IMediator mediator, [FromQuery] string path, [FromQuery]SyncPeriod period) + private static IResult DownloadNotifications([FromServices] IWebHostEnvironment env, string id) { - var result = await mediator.Send(new DownloadCommand() { Type = typeof(ImportNotification), Path = path, SyncPeriod = period }); - return Results.File(result.Zip, "application/zip", "notifications.zip"); + var stream = File.OpenRead($"{System.IO.Path.Combine(env.ContentRootPath, id)}.zip"); + return Results.File(stream, "application/zip", $"{id}.zip"); } - private static async Task DownloadClearanceRequests([FromServices] IMediator mediator, [FromQuery] string path, [FromQuery] SyncPeriod period) + private static async Task GenerateDownload([FromServices] ICdmsMediator mediator, [FromQuery] SyncPeriod period) { - var result = await mediator.Send(new DownloadCommand() { Type = typeof(AlvsClearanceRequest), Path = path, SyncPeriod = period }); - return Results.File(result.Zip, "application/zip", "clearancerequests.zip"); + var command = new DownloadCommand() { SyncPeriod = period }; + await mediator.SendJob(command); + return Results.Ok(command.JobId); } private static Task GetAllSyncJobs([FromServices] ISyncJobStore store) diff --git a/CdmsBackend/Mediatr/CdmsMediator.cs b/CdmsBackend/Mediatr/CdmsMediator.cs index 213411d..56d5c42 100644 --- a/CdmsBackend/Mediatr/CdmsMediator.cs +++ b/CdmsBackend/Mediatr/CdmsMediator.cs @@ -30,6 +30,20 @@ await backgroundTaskQueue.QueueBackgroundWorkItemAsync(async (ct) => }); } + public async Task SendJob(TRequest request, CancellationToken cancellationToken = default) where TRequest : IRequest, ISyncJob + { + var job = syncJobStore.CreateJob(request.JobId, request.Timespan, request.Resource); + + await backgroundTaskQueue.QueueBackgroundWorkItemAsync(async (ct) => + { + using var scope = serviceScopeFactory.CreateScope(); + using var activity = ActivitySource.StartActivity(ActivityName, ActivityKind.Client); + var m = scope.ServiceProvider.GetRequiredService(); + await m.Send(request, job.CancellationToken); + job.Complete(); + }); + } + Task ICdmsMediator.Send(IRequest request, CancellationToken cancellationToken) { return mediator.Send(request, cancellationToken); diff --git a/CdmsBackend/Mediatr/ICdmsMediator.cs b/CdmsBackend/Mediatr/ICdmsMediator.cs index fd4b5ea..6443f1b 100644 --- a/CdmsBackend/Mediatr/ICdmsMediator.cs +++ b/CdmsBackend/Mediatr/ICdmsMediator.cs @@ -1,4 +1,4 @@ -using Cdms.SyncJob; +using Cdms.SyncJob; using MediatR; namespace CdmsBackend.Mediatr; @@ -8,6 +8,9 @@ public interface ICdmsMediator Task SendSyncJob(TRequest request, CancellationToken cancellationToken = default) where TRequest : IRequest, ISyncJob; + Task SendJob(TRequest request, CancellationToken cancellationToken = default) + where TRequest : IRequest, ISyncJob; + /// From d926789aa5ec788e938515b7b7676816d3aa794e Mon Sep 17 00:00:00 2001 From: Thomas Anderson Date: Tue, 3 Dec 2024 16:40:19 +0000 Subject: [PATCH 5/5] Added some tests against the import notification, and verified that the json is the same after the flatten, and unflattening --- Cdms.SensitiveData/SensitiveDataSerializer.cs | 12 +++++- .../SensitiveDataTests.cs | 43 +++++++++++++++++++ 2 files changed, 53 insertions(+), 2 deletions(-) create mode 100644 CdmsBackend.IntegrationTests/SensitiveDataTests.cs diff --git a/Cdms.SensitiveData/SensitiveDataSerializer.cs b/Cdms.SensitiveData/SensitiveDataSerializer.cs index 1a2a8d2..8c22247 100644 --- a/Cdms.SensitiveData/SensitiveDataSerializer.cs +++ b/Cdms.SensitiveData/SensitiveDataSerializer.cs @@ -48,7 +48,12 @@ public T Deserialize(string json, Action optionsOverri public string RedactRawJson(string json, Type type) { + if (options.Value.Include) + { + return json; + } var sensitiveFields = SensitiveFieldsProvider.Get(type); + var jObject = JObject.Parse(json); var fields = jObject.Flatten(); @@ -57,7 +62,10 @@ public string RedactRawJson(string json, Type type) { if (fields.TryGetValue(field, out var value)) { - fields[field] = options.Value.Getter(value.ToString()!); + if (!options.Value.Include) + { + fields[field] = options.Value.Getter(value.ToString()!); + } } else { @@ -65,7 +73,7 @@ public string RedactRawJson(string json, Type type) { var key = fields.Keys.ElementAt(i); var replaced = Regex.Replace(key, "\\[.*?\\]", "", RegexOptions.NonBacktracking); - if (replaced == field && fields.TryGetValue(key, out var v)) + if (replaced == field && fields.TryGetValue(key, out var v) && !options.Value.Include) { fields[key] = options.Value.Getter(v.ToString()!); } diff --git a/CdmsBackend.IntegrationTests/SensitiveDataTests.cs b/CdmsBackend.IntegrationTests/SensitiveDataTests.cs new file mode 100644 index 0000000..d30e94b --- /dev/null +++ b/CdmsBackend.IntegrationTests/SensitiveDataTests.cs @@ -0,0 +1,43 @@ +using System.Text.Json.Nodes; +using Cdms.SensitiveData; +using Cdms.Types.Ipaffs; +using FluentAssertions; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Xunit; + +namespace CdmsBackend.IntegrationTests; + +public class SensitiveDataTests +{ + [Fact] + public void WhenIncludeSensitiveData_RedactedShouldBeSameAsJson() + { + string json = + File.ReadAllText(Path.GetFullPath("..\\..\\..\\Fixtures\\SmokeTest\\IPAFFS\\CHEDA\\CHEDA_GB_2024_1041389-ee0e6fcf-52a4-45ea-8830-d4553ee70361.json")); + + SensitiveDataOptions options = new SensitiveDataOptions { Getter = s => "TestRedacted", Include = true }; + var serializer = new SensitiveDataSerializer(Options.Create(options), NullLogger.Instance); + + var result = serializer.RedactRawJson(json, typeof(ImportNotification)); + + JsonNode.DeepEquals(JsonNode.Parse(json), JsonNode.Parse(result)).Should().BeTrue(); + + } + + [Fact] + public void WhenIncludeSensitiveData_RedactedShouldBeDifferentJson() + { + string json = + File.ReadAllText(Path.GetFullPath("..\\..\\..\\Fixtures\\SmokeTest\\IPAFFS\\CHEDA\\CHEDA_GB_2024_1041389-ee0e6fcf-52a4-45ea-8830-d4553ee70361.json")); + + SensitiveDataOptions options = new SensitiveDataOptions { Getter = s => "TestRedacted", Include = false }; + var serializer = new SensitiveDataSerializer(Options.Create(options), NullLogger.Instance); + + var result = serializer.RedactRawJson(json, typeof(ImportNotification)); + + JsonNode.DeepEquals(JsonNode.Parse(json), JsonNode.Parse(result)).Should().BeFalse(); + result.Should().Contain("TestRedacted"); + + } +} \ No newline at end of file