diff --git a/Cdms.Business/Commands/DownloadNotificationsCommand.cs b/Cdms.Business/Commands/DownloadNotificationsCommand.cs new file mode 100644 index 0000000..be241fd --- /dev/null +++ b/Cdms.Business/Commands/DownloadNotificationsCommand.cs @@ -0,0 +1,77 @@ +using System.Dynamic; +using System.IO; +using System.IO.Compression; +using System.Text.Json.Serialization; +using Bogus; +using Cdms.BlobService; +using System.Threading; +using MediatR; +using Newtonsoft.Json; +using JsonSerializer = System.Text.Json.JsonSerializer; +using SharpCompress.Writers; +using Cdms.SensitiveData; +using Cdms.Types.Ipaffs; +using Microsoft.AspNetCore.Hosting; +using Cdms.SyncJob; +using Cdms.Types.Alvs; +using Cdms.Types.Gvms; + +namespace Cdms.Business.Commands; + +public class DownloadCommand : IRequest, ISyncJob +{ + [System.Text.Json.Serialization.JsonConverter(typeof(JsonStringEnumConverter))] + public SyncPeriod SyncPeriod { get; set; } + + public Guid JobId { get; } = Guid.NewGuid(); + public string Timespan { get; } = null!; + public string Resource { get; } = null!; + + internal class Handler(IBlobService blobService, ISensitiveDataSerializer sensitiveDataSerializer, IWebHostEnvironment env) : IRequestHandler + { + + public async Task Handle(DownloadCommand request, CancellationToken cancellationToken) + { + string subFolder = $"temp\\{request.JobId}"; + string rootFolder = Path.Combine(env.ContentRootPath, subFolder); + Directory.CreateDirectory(rootFolder); + + await Download(request, rootFolder, "RAW/IPAFFS/CHEDA", typeof(ImportNotification), cancellationToken); + await Download(request, rootFolder, "RAW/IPAFFS/CHEDD", typeof(ImportNotification), cancellationToken); + await Download(request, rootFolder, "RAW/IPAFFS/CHEDP", typeof(ImportNotification), cancellationToken); + await Download(request, rootFolder, "RAW/IPAFFS/CHEDPP", typeof(ImportNotification), cancellationToken); + + await Download(request, rootFolder, "RAW/ALVS", typeof(AlvsClearanceRequest), cancellationToken); + + await Download(request, rootFolder, "RAW/GVMSAPIRESPONSE", typeof(SearchGmrsForDeclarationIdsResponse), cancellationToken); + + await Download(request, rootFolder, "RAW/DECISIONS", typeof(AlvsClearanceRequest), cancellationToken); + + ZipFile.CreateFromDirectory(rootFolder, $"{env.ContentRootPath}\\{request.JobId}.zip"); + + Directory.Delete(rootFolder, true); + } + + private async Task Download(DownloadCommand request, string rootFolder, string folder, Type type, CancellationToken cancellationToken) + { + + ParallelOptions options = new() { CancellationToken = cancellationToken, MaxDegreeOfParallelism = 10 }; + var result = blobService.GetResourcesAsync($"{folder}{request.SyncPeriod.GetPeriodPath()}", cancellationToken); + + //Write local files + await Parallel.ForEachAsync(result, options, async (item, token) => + { + var blobContent = await blobService.GetResource(item, cancellationToken); + string redactedContent = sensitiveDataSerializer.RedactRawJson(blobContent, type); + var filename = System.IO.Path.Combine(rootFolder, item.Name.Replace('/', System.IO.Path.DirectorySeparatorChar)); + Directory.CreateDirectory(System.IO.Path.GetDirectoryName(filename)!); + await File.WriteAllTextAsync(filename, redactedContent, cancellationToken); + }); + } + + } + + public record Result(byte[] Zip); + + +} \ No newline at end of file diff --git a/Cdms.Business/Commands/SyncHandler.cs b/Cdms.Business/Commands/SyncHandler.cs index e374692..667751c 100644 --- a/Cdms.Business/Commands/SyncHandler.cs +++ b/Cdms.Business/Commands/SyncHandler.cs @@ -103,7 +103,7 @@ await Parallel.ForEachAsync(paths, protected async Task SyncBlobPath(string path, SyncPeriod period, string topic, SyncJob.SyncJob job, CancellationToken cancellationToken) { - var result = blobService.GetResourcesAsync($"{path}{GetPeriodPath(period)}", cancellationToken); + var result = blobService.GetResourcesAsync($"{path}{period.GetPeriodPath()}", cancellationToken); await Parallel.ForEachAsync(result, new ParallelOptions() { CancellationToken = cancellationToken, MaxDegreeOfParallelism = maxDegreeOfParallelism }, async (item, token) => { @@ -184,31 +184,5 @@ await bus.Publish(message, } } } - - - - private static string GetPeriodPath(SyncPeriod period) - { - if (period == SyncPeriod.LastMonth) - { - return DateTime.Today.AddMonths(-1).ToString("/yyyy/MM/"); - } - else if (period == SyncPeriod.ThisMonth) - { - return DateTime.Today.ToString("/yyyy/MM/"); - } - else if (period == SyncPeriod.Today) - { - return DateTime.Today.ToString("/yyyy/MM/dd/"); - } - else if (period == SyncPeriod.All) - { - return "/"; - } - else - { - throw new ArgumentException($"Unexpected SyncPeriod {period}"); - } - } } } \ No newline at end of file diff --git a/Cdms.Business/Commands/SyncPeriodExtensions.cs b/Cdms.Business/Commands/SyncPeriodExtensions.cs new file mode 100644 index 0000000..51bb13f --- /dev/null +++ b/Cdms.Business/Commands/SyncPeriodExtensions.cs @@ -0,0 +1,28 @@ +namespace Cdms.Business.Commands; + +public static class SyncPeriodExtensions +{ + public static string GetPeriodPath(this SyncPeriod period) + { + if (period == SyncPeriod.LastMonth) + { + return DateTime.Today.AddMonths(-1).ToString("/yyyy/MM/"); + } + else if (period == SyncPeriod.ThisMonth) + { + return DateTime.Today.ToString("/yyyy/MM/"); + } + else if (period == SyncPeriod.Today) + { + return DateTime.Today.ToString("/yyyy/MM/dd/"); + } + else if (period == SyncPeriod.All) + { + return "/"; + } + else + { + throw new ArgumentException($"Unexpected SyncPeriod {period}"); + } + } +} \ No newline at end of file diff --git a/Cdms.SensitiveData.Tests/SensitiveDataSerializerTests.cs b/Cdms.SensitiveData.Tests/SensitiveDataSerializerTests.cs index b983e54..c1ec848 100644 --- a/Cdms.SensitiveData.Tests/SensitiveDataSerializerTests.cs +++ b/Cdms.SensitiveData.Tests/SensitiveDataSerializerTests.cs @@ -67,4 +67,35 @@ public void WhenIncludeSensitiveData_ThenDataShouldNotBeRedacted() result.SimpleStringArrayTwo[0].Should().Be("Test String Array Two Item One"); result.SimpleStringArrayTwo[1].Should().Be("Test String Array Two Item Two"); } + + [Fact] + public void WhenDoNotIncludeSensitiveData_AndRequestForRawJson_ThenDataShouldBeRedacted() + { + // ARRANGE + SensitiveDataOptions options = new SensitiveDataOptions { Getter = s => "TestRedacted", Include = false }; + var serializer = new SensitiveDataSerializer(Options.Create(options), NullLogger.Instance); + + var simpleClass = new SimpleClass() + { + SimpleStringOne = "Test String One", + SimpleStringTwo = "Test String Two", + SimpleStringArrayOne = + new[] { "Test String Array One Item One", "Test String Array One Item Two" }, + SimpleStringArrayTwo = new[] { "Test String Array Two Item One", "Test String Array Two Item Two" } + }; + + var json = JsonSerializer.Serialize(simpleClass, new JsonSerializerOptions() { PropertyNamingPolicy = JsonNamingPolicy.CamelCase}); + + // ACT + var result = serializer.RedactRawJson(json, typeof(SimpleClass)); + + // ASSERT + var resultClass = JsonSerializer.Deserialize(result, new JsonSerializerOptions() { PropertyNameCaseInsensitive = true }); + resultClass?.SimpleStringOne.Should().Be("TestRedacted"); + resultClass?.SimpleStringTwo.Should().Be("Test String Two"); + resultClass?.SimpleStringArrayOne[0].Should().Be("TestRedacted"); + resultClass?.SimpleStringArrayOne[1].Should().Be("TestRedacted"); + resultClass?.SimpleStringArrayTwo[0].Should().Be("Test String Array Two Item One"); + resultClass?.SimpleStringArrayTwo[1].Should().Be("Test String Array Two Item Two"); + } } \ No newline at end of file diff --git a/Cdms.SensitiveData/Cdms.SensitiveData.csproj b/Cdms.SensitiveData/Cdms.SensitiveData.csproj index 8868104..2b9ff2a 100644 --- a/Cdms.SensitiveData/Cdms.SensitiveData.csproj +++ b/Cdms.SensitiveData/Cdms.SensitiveData.csproj @@ -8,14 +8,20 @@ + + + + + + diff --git a/Cdms.SensitiveData/ISensitiveDataSerializer.cs b/Cdms.SensitiveData/ISensitiveDataSerializer.cs index 8056a84..e745d54 100644 --- a/Cdms.SensitiveData/ISensitiveDataSerializer.cs +++ b/Cdms.SensitiveData/ISensitiveDataSerializer.cs @@ -5,4 +5,7 @@ namespace Cdms.SensitiveData; public interface ISensitiveDataSerializer { public T Deserialize(string json, Action optionsOverride = null!); -} \ No newline at end of file + + string RedactRawJson(string json, Type type); +} + diff --git a/Cdms.SensitiveData/SensitiveDataSerializer.cs b/Cdms.SensitiveData/SensitiveDataSerializer.cs index c129ed4..8c22247 100644 --- a/Cdms.SensitiveData/SensitiveDataSerializer.cs +++ b/Cdms.SensitiveData/SensitiveDataSerializer.cs @@ -1,7 +1,11 @@ using System.Text.Json; using System.Text.Json.Serialization; +using System.Text.RegularExpressions; +using JsonFlatten; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; +using Newtonsoft.Json.Linq; +using Newtonsoft.Json.Serialization; namespace Cdms.SensitiveData; @@ -41,4 +45,43 @@ public T Deserialize(string json, Action optionsOverri } } + + public string RedactRawJson(string json, Type type) + { + if (options.Value.Include) + { + return json; + } + var sensitiveFields = SensitiveFieldsProvider.Get(type); + + var jObject = JObject.Parse(json); + + var fields = jObject.Flatten(); + + foreach (var field in sensitiveFields) + { + if (fields.TryGetValue(field, out var value)) + { + if (!options.Value.Include) + { + fields[field] = options.Value.Getter(value.ToString()!); + } + } + else + { + for (int i = 0; i < fields.Keys.Count; i++) + { + var key = fields.Keys.ElementAt(i); + var replaced = Regex.Replace(key, "\\[.*?\\]", "", RegexOptions.NonBacktracking); + if (replaced == field && fields.TryGetValue(key, out var v) && !options.Value.Include) + { + fields[key] = options.Value.Getter(v.ToString()!); + } + } + } + } + + var redactedString = fields.Unflatten().ToString(); + return redactedString; + } } \ No newline at end of file diff --git a/Cdms.SensitiveData/SensitiveFieldsProvider.cs b/Cdms.SensitiveData/SensitiveFieldsProvider.cs new file mode 100644 index 0000000..3081b35 --- /dev/null +++ b/Cdms.SensitiveData/SensitiveFieldsProvider.cs @@ -0,0 +1,92 @@ +using System.Reflection; +using System.Text.Json; + +namespace Cdms.SensitiveData; + +public static class SensitiveFieldsProvider +{ + private static Dictionary> cache = new(); + private static readonly object cacheLock = new (); + public static List Get() + { + lock (cacheLock) + { + if (cache.TryGetValue(typeof(T), out var value)) + { + return value; + } + + var type = typeof(T); + + var list = GetSensitiveFields(string.Empty, type); + cache.Add(typeof(T), list); + return list; + } + + + } + + public static List Get(Type type) + { + lock (cacheLock) + { + if (cache.TryGetValue(type, out var value)) + { + return value; + } + + var list = GetSensitiveFields(string.Empty, type); + cache.Add(type, list); + return list; + } + + + } + + private static List GetSensitiveFields(string root, Type type) + { + var namingPolicy = JsonNamingPolicy.CamelCase; + var list = new List(); + foreach (var property in type.GetProperties()) + { + string currentPath; + currentPath = string.IsNullOrEmpty(root) ? $"{namingPolicy.ConvertName(property.Name)}" : $"{namingPolicy.ConvertName(root)}.{namingPolicy.ConvertName(property.Name)}"; + + if (property.CustomAttributes.Any(x => x.AttributeType == typeof(SensitiveDataAttribute))) + { + list.Add(currentPath); + } + else + { + Type elementType = GetElementType(property)!; + + if (elementType != null && elementType.Namespace != "System") + { + list.AddRange(GetSensitiveFields($"{currentPath}", elementType!)); + } + } + } + + return list; + } + + private static Type? GetElementType(PropertyInfo property) + { + if (property.PropertyType.IsArray) + { + return property.PropertyType.GetElementType()!; + } + else if (property.PropertyType.IsGenericType) + { + return property.PropertyType.GetGenericArguments()[0]; + + } + else if (property.PropertyType.IsClass) + { + return property.PropertyType; + + } + + return default; + } +} \ No newline at end of file diff --git a/CdmsBackend.IntegrationTests/SensitiveDataTests.cs b/CdmsBackend.IntegrationTests/SensitiveDataTests.cs new file mode 100644 index 0000000..d30e94b --- /dev/null +++ b/CdmsBackend.IntegrationTests/SensitiveDataTests.cs @@ -0,0 +1,43 @@ +using System.Text.Json.Nodes; +using Cdms.SensitiveData; +using Cdms.Types.Ipaffs; +using FluentAssertions; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Xunit; + +namespace CdmsBackend.IntegrationTests; + +public class SensitiveDataTests +{ + [Fact] + public void WhenIncludeSensitiveData_RedactedShouldBeSameAsJson() + { + string json = + File.ReadAllText(Path.GetFullPath("..\\..\\..\\Fixtures\\SmokeTest\\IPAFFS\\CHEDA\\CHEDA_GB_2024_1041389-ee0e6fcf-52a4-45ea-8830-d4553ee70361.json")); + + SensitiveDataOptions options = new SensitiveDataOptions { Getter = s => "TestRedacted", Include = true }; + var serializer = new SensitiveDataSerializer(Options.Create(options), NullLogger.Instance); + + var result = serializer.RedactRawJson(json, typeof(ImportNotification)); + + JsonNode.DeepEquals(JsonNode.Parse(json), JsonNode.Parse(result)).Should().BeTrue(); + + } + + [Fact] + public void WhenIncludeSensitiveData_RedactedShouldBeDifferentJson() + { + string json = + File.ReadAllText(Path.GetFullPath("..\\..\\..\\Fixtures\\SmokeTest\\IPAFFS\\CHEDA\\CHEDA_GB_2024_1041389-ee0e6fcf-52a4-45ea-8830-d4553ee70361.json")); + + SensitiveDataOptions options = new SensitiveDataOptions { Getter = s => "TestRedacted", Include = false }; + var serializer = new SensitiveDataSerializer(Options.Create(options), NullLogger.Instance); + + var result = serializer.RedactRawJson(json, typeof(ImportNotification)); + + JsonNode.DeepEquals(JsonNode.Parse(json), JsonNode.Parse(result)).Should().BeFalse(); + result.Should().Contain("TestRedacted"); + + } +} \ No newline at end of file diff --git a/CdmsBackend/Endpoints/SyncEndpoints.cs b/CdmsBackend/Endpoints/SyncEndpoints.cs index ae09ac1..b5f51b1 100644 --- a/CdmsBackend/Endpoints/SyncEndpoints.cs +++ b/CdmsBackend/Endpoints/SyncEndpoints.cs @@ -1,10 +1,17 @@ +using System.IO.Compression; +using Azure.Storage.Blobs; +using Azure.Storage.Blobs.Models; using Cdms.Business.Commands; using Cdms.Consumers.MemoryQueue; using Cdms.SyncJob; +using Cdms.Types.Alvs; +using Cdms.Types.Ipaffs; using CdmsBackend.Config; using CdmsBackend.Mediatr; +using MediatR; using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Options; +using static Cdms.Business.Commands.DownloadCommand; namespace CdmsBackend.Endpoints; @@ -20,6 +27,9 @@ public static void UseSyncEndpoints(this IEndpointRouteBuilder app, IOptions InitialiseEnvironment(IHost app, SyncPeriod return Results.Ok(); } + private static IResult DownloadNotifications([FromServices] IWebHostEnvironment env, string id) + { + var stream = File.OpenRead($"{System.IO.Path.Combine(env.ContentRootPath, id)}.zip"); + return Results.File(stream, "application/zip", $"{id}.zip"); + } + + private static async Task GenerateDownload([FromServices] ICdmsMediator mediator, [FromQuery] SyncPeriod period) + { + var command = new DownloadCommand() { SyncPeriod = period }; + await mediator.SendJob(command); + return Results.Ok(command.JobId); + } + private static Task GetAllSyncJobs([FromServices] ISyncJobStore store) { return Task.FromResult(Results.Ok(new { jobs = store.GetJobs() })); diff --git a/CdmsBackend/Mediatr/CdmsMediator.cs b/CdmsBackend/Mediatr/CdmsMediator.cs index 213411d..56d5c42 100644 --- a/CdmsBackend/Mediatr/CdmsMediator.cs +++ b/CdmsBackend/Mediatr/CdmsMediator.cs @@ -30,6 +30,20 @@ await backgroundTaskQueue.QueueBackgroundWorkItemAsync(async (ct) => }); } + public async Task SendJob(TRequest request, CancellationToken cancellationToken = default) where TRequest : IRequest, ISyncJob + { + var job = syncJobStore.CreateJob(request.JobId, request.Timespan, request.Resource); + + await backgroundTaskQueue.QueueBackgroundWorkItemAsync(async (ct) => + { + using var scope = serviceScopeFactory.CreateScope(); + using var activity = ActivitySource.StartActivity(ActivityName, ActivityKind.Client); + var m = scope.ServiceProvider.GetRequiredService(); + await m.Send(request, job.CancellationToken); + job.Complete(); + }); + } + Task ICdmsMediator.Send(IRequest request, CancellationToken cancellationToken) { return mediator.Send(request, cancellationToken); diff --git a/CdmsBackend/Mediatr/ICdmsMediator.cs b/CdmsBackend/Mediatr/ICdmsMediator.cs index fd4b5ea..6443f1b 100644 --- a/CdmsBackend/Mediatr/ICdmsMediator.cs +++ b/CdmsBackend/Mediatr/ICdmsMediator.cs @@ -1,4 +1,4 @@ -using Cdms.SyncJob; +using Cdms.SyncJob; using MediatR; namespace CdmsBackend.Mediatr; @@ -8,6 +8,9 @@ public interface ICdmsMediator Task SendSyncJob(TRequest request, CancellationToken cancellationToken = default) where TRequest : IRequest, ISyncJob; + Task SendJob(TRequest request, CancellationToken cancellationToken = default) + where TRequest : IRequest, ISyncJob; + ///