Merge pull request #59 from DEFRA/feature/cdms-165

Added the ability to redact data without Deserializing to an object.

Cdms.Business/Commands/DownloadNotificationsCommand.cs

@@ -0,0 +1,77 @@
+using System.Dynamic;
+using System.IO;
+using System.IO.Compression;
+using System.Text.Json.Serialization;
+using Bogus;
+using Cdms.BlobService;
+using System.Threading;
+using MediatR;
+using Newtonsoft.Json;
+using JsonSerializer = System.Text.Json.JsonSerializer;
+using SharpCompress.Writers;
+using Cdms.SensitiveData;
+using Cdms.Types.Ipaffs;
+using Microsoft.AspNetCore.Hosting;
+using Cdms.SyncJob;
+using Cdms.Types.Alvs;
+using Cdms.Types.Gvms;
+
+namespace Cdms.Business.Commands;
+
+public class DownloadCommand : IRequest, ISyncJob
+{
+    [System.Text.Json.Serialization.JsonConverter(typeof(JsonStringEnumConverter<SyncPeriod>))]
+    public SyncPeriod SyncPeriod { get; set; }
+
+    public Guid JobId { get; } = Guid.NewGuid();
+    public string Timespan { get; } = null!;
+    public string Resource { get; } = null!;
+
+    internal class Handler(IBlobService blobService, ISensitiveDataSerializer sensitiveDataSerializer, IWebHostEnvironment env) : IRequestHandler<DownloadCommand>
+    {
+
+        public async Task Handle(DownloadCommand request, CancellationToken cancellationToken)
+        {
+            string subFolder = $"temp\\{request.JobId}";
+            string rootFolder = Path.Combine(env.ContentRootPath, subFolder);
+            Directory.CreateDirectory(rootFolder);
+
+            await Download(request, rootFolder, "RAW/IPAFFS/CHEDA", typeof(ImportNotification), cancellationToken);
+            await Download(request, rootFolder, "RAW/IPAFFS/CHEDD", typeof(ImportNotification), cancellationToken);
+            await Download(request, rootFolder, "RAW/IPAFFS/CHEDP", typeof(ImportNotification), cancellationToken);
+            await Download(request, rootFolder, "RAW/IPAFFS/CHEDPP", typeof(ImportNotification), cancellationToken);
+
+            await Download(request, rootFolder, "RAW/ALVS", typeof(AlvsClearanceRequest), cancellationToken);
+
+            await Download(request, rootFolder, "RAW/GVMSAPIRESPONSE", typeof(SearchGmrsForDeclarationIdsResponse), cancellationToken);
+
+            await Download(request, rootFolder, "RAW/DECISIONS", typeof(AlvsClearanceRequest), cancellationToken);
+
+            ZipFile.CreateFromDirectory(rootFolder, $"{env.ContentRootPath}\\{request.JobId}.zip");
+
+            Directory.Delete(rootFolder, true);
+        }
+
+        private async Task Download(DownloadCommand request, string rootFolder, string folder, Type type, CancellationToken cancellationToken)
+        {
+
+            ParallelOptions options = new() { CancellationToken = cancellationToken, MaxDegreeOfParallelism = 10 };
+            var result = blobService.GetResourcesAsync($"{folder}{request.SyncPeriod.GetPeriodPath()}", cancellationToken);
+
+            //Write local files
+            await Parallel.ForEachAsync(result, options, async (item, token) =>
+            {
+                var blobContent = await blobService.GetResource(item, cancellationToken);
+                string redactedContent = sensitiveDataSerializer.RedactRawJson(blobContent, type);
+                var filename = System.IO.Path.Combine(rootFolder, item.Name.Replace('/', System.IO.Path.DirectorySeparatorChar));
+                Directory.CreateDirectory(System.IO.Path.GetDirectoryName(filename)!);
+                await File.WriteAllTextAsync(filename, redactedContent, cancellationToken);
+            });
+        }
+
+    }
+
+    public record Result(byte[] Zip);
+
+
+}

Cdms.Business/Commands/SyncHandler.cs

@@ -103,7 +103,7 @@ await Parallel.ForEachAsync(paths,
         protected async Task SyncBlobPath<TRequest>(string path, SyncPeriod period, string topic, SyncJob.SyncJob job,
             CancellationToken cancellationToken)
         {
-            var result = blobService.GetResourcesAsync($"{path}{GetPeriodPath(period)}", cancellationToken);
+            var result = blobService.GetResourcesAsync($"{path}{period.GetPeriodPath()}", cancellationToken);
 
             await Parallel.ForEachAsync(result, new ParallelOptions() { CancellationToken = cancellationToken, MaxDegreeOfParallelism = maxDegreeOfParallelism }, async (item, token) =>
             {
@@ -184,31 +184,5 @@ await bus.Publish(message,
                 }
             }
         }
-
-
-
-        private static string GetPeriodPath(SyncPeriod period)
-        {
-            if (period == SyncPeriod.LastMonth)
-            {
-                return DateTime.Today.AddMonths(-1).ToString("/yyyy/MM/");
-            }
-            else if (period == SyncPeriod.ThisMonth)
-            {
-                return DateTime.Today.ToString("/yyyy/MM/");
-            }
-            else if (period == SyncPeriod.Today)
-            {
-                return DateTime.Today.ToString("/yyyy/MM/dd/");
-            }
-            else if (period == SyncPeriod.All)
-            {
-                return "/";
-            }
-            else
-            {
-                throw new ArgumentException($"Unexpected SyncPeriod {period}");
-            }
-        }
     }
 }

Cdms.Business/Commands/SyncPeriodExtensions.cs

@@ -0,0 +1,28 @@
+namespace Cdms.Business.Commands;
+
+public static class SyncPeriodExtensions
+{
+    public static string GetPeriodPath(this SyncPeriod period)
+    {
+        if (period == SyncPeriod.LastMonth)
+        {
+            return DateTime.Today.AddMonths(-1).ToString("/yyyy/MM/");
+        }
+        else if (period == SyncPeriod.ThisMonth)
+        {
+            return DateTime.Today.ToString("/yyyy/MM/");
+        }
+        else if (period == SyncPeriod.Today)
+        {
+            return DateTime.Today.ToString("/yyyy/MM/dd/");
+        }
+        else if (period == SyncPeriod.All)
+        {
+            return "/";
+        }
+        else
+        {
+            throw new ArgumentException($"Unexpected SyncPeriod {period}");
+        }
+    }
+}

Cdms.SensitiveData.Tests/SensitiveDataSerializerTests.cs

@@ -67,4 +67,35 @@ public void WhenIncludeSensitiveData_ThenDataShouldNotBeRedacted()
         result.SimpleStringArrayTwo[0].Should().Be("Test String Array Two Item One");
         result.SimpleStringArrayTwo[1].Should().Be("Test String Array Two Item Two");
     }
+
+    [Fact]
+    public void WhenDoNotIncludeSensitiveData_AndRequestForRawJson_ThenDataShouldBeRedacted()
+    {
+        // ARRANGE
+        SensitiveDataOptions options = new SensitiveDataOptions { Getter = s => "TestRedacted", Include = false };
+        var serializer = new SensitiveDataSerializer(Options.Create(options), NullLogger<SensitiveDataSerializer>.Instance);
+
+        var simpleClass = new SimpleClass()
+        {
+            SimpleStringOne = "Test String One",
+            SimpleStringTwo = "Test String Two",
+            SimpleStringArrayOne =
+                new[] { "Test String Array One Item One", "Test String Array One Item Two" },
+            SimpleStringArrayTwo = new[] { "Test String Array Two Item One", "Test String Array Two Item Two" }
+        };
+
+        var json = JsonSerializer.Serialize(simpleClass, new JsonSerializerOptions() { PropertyNamingPolicy = JsonNamingPolicy.CamelCase});
+
+        // ACT
+        var result = serializer.RedactRawJson(json, typeof(SimpleClass));
+
+        // ASSERT
+        var resultClass = JsonSerializer.Deserialize<SimpleClass>(result, new JsonSerializerOptions() { PropertyNameCaseInsensitive = true });
+        resultClass?.SimpleStringOne.Should().Be("TestRedacted");
+        resultClass?.SimpleStringTwo.Should().Be("Test String Two");
+        resultClass?.SimpleStringArrayOne[0].Should().Be("TestRedacted");
+        resultClass?.SimpleStringArrayOne[1].Should().Be("TestRedacted");
+        resultClass?.SimpleStringArrayTwo[0].Should().Be("Test String Array Two Item One");
+        resultClass?.SimpleStringArrayTwo[1].Should().Be("Test String Array Two Item Two");
+    }
 }

Cdms.SensitiveData/Cdms.SensitiveData.csproj

@@ -8,14 +8,20 @@
   </PropertyGroup>
 
   <ItemGroup>
+    <PackageReference Include="JsonFlatten" Version="1.0.4" />
     <PackageReference Include="Microsoft.Extensions.Configuration" Version="8.0.0" />
     <PackageReference Include="Microsoft.Extensions.Configuration.Abstractions" Version="8.0.0" />
     <PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="8.0.2" />
     <PackageReference Include="Microsoft.Extensions.Options" Version="8.0.0" />
+    <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
   </ItemGroup>
 
   <ItemGroup>
     <ProjectReference Include="..\Cdms.Common\Cdms.Common.csproj" />
   </ItemGroup>
 
+  <ItemGroup>
+    <Folder Include="Flattener\" />
+  </ItemGroup>
+
 </Project>

Cdms.SensitiveData/ISensitiveDataSerializer.cs

@@ -5,4 +5,7 @@ namespace Cdms.SensitiveData;
 public interface ISensitiveDataSerializer
 {
     public T Deserialize<T>(string json, Action<JsonSerializerOptions> optionsOverride = null!);
-}
+
+    string RedactRawJson(string json, Type type);
+}
+

Cdms.SensitiveData/SensitiveDataSerializer.cs

@@ -1,7 +1,11 @@
 using System.Text.Json;
 using System.Text.Json.Serialization;
+using System.Text.RegularExpressions;
+using JsonFlatten;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
+using Newtonsoft.Json.Linq;
+using Newtonsoft.Json.Serialization;
 
 namespace Cdms.SensitiveData;
 
@@ -41,4 +45,43 @@ public T Deserialize<T>(string json, Action<JsonSerializerOptions> optionsOverri
         }
 
     }
+
+    public string RedactRawJson(string json, Type type)
+    {
+        if (options.Value.Include)
+        {
+            return json;
+        }
+        var sensitiveFields = SensitiveFieldsProvider.Get(type);
+
+        var jObject = JObject.Parse(json);
+
+        var fields = jObject.Flatten();
+
+        foreach (var field in sensitiveFields)
+        {
+            if (fields.TryGetValue(field, out var value))
+            {
+                if (!options.Value.Include)
+                {
+                    fields[field] = options.Value.Getter(value.ToString()!);
+                }
+            }
+            else
+            {
+                for (int i = 0; i < fields.Keys.Count; i++)
+                {
+                    var key = fields.Keys.ElementAt(i);
+                    var replaced = Regex.Replace(key, "\\[.*?\\]", "", RegexOptions.NonBacktracking);
+                    if (replaced == field && fields.TryGetValue(key, out var v) && !options.Value.Include)
+                    {
+                        fields[key] = options.Value.Getter(v.ToString()!);
+                    }
+                }
+            }
+        }
+
+        var redactedString = fields.Unflatten().ToString();
+        return redactedString;
+    }
 }

Cdms.SensitiveData/SensitiveFieldsProvider.cs

@@ -0,0 +1,92 @@
+using System.Reflection;
+using System.Text.Json;
+
+namespace Cdms.SensitiveData;
+
+public static class SensitiveFieldsProvider
+{
+    private static Dictionary<Type, List<string>> cache = new();
+    private static readonly object cacheLock = new ();
+    public static List<string> Get<T>()
+    {
+        lock (cacheLock)
+        {
+            if (cache.TryGetValue(typeof(T), out var value))
+            {
+                return value;
+            }
+
+            var type = typeof(T);
+
+            var list = GetSensitiveFields(string.Empty, type);
+            cache.Add(typeof(T), list);
+            return list;
+        }
+
+
+    }
+
+    public static List<string> Get(Type type)
+    {
+        lock (cacheLock)
+        {
+            if (cache.TryGetValue(type, out var value))
+            {
+                return value;
+            }
+
+            var list = GetSensitiveFields(string.Empty, type);
+            cache.Add(type, list);
+            return list;
+        }
+
+
+    }
+
+    private static List<string> GetSensitiveFields(string root, Type type)
+    {
+        var namingPolicy = JsonNamingPolicy.CamelCase;
+        var list = new List<string>();
+        foreach (var property in type.GetProperties())
+        {
+            string currentPath;
+            currentPath = string.IsNullOrEmpty(root) ? $"{namingPolicy.ConvertName(property.Name)}" : $"{namingPolicy.ConvertName(root)}.{namingPolicy.ConvertName(property.Name)}";
+
+            if (property.CustomAttributes.Any(x => x.AttributeType == typeof(SensitiveDataAttribute)))
+            {
+                list.Add(currentPath);
+            }
+            else
+            {
+                Type elementType = GetElementType(property)!;
+
+                if (elementType != null && elementType.Namespace != "System")
+                {
+                    list.AddRange(GetSensitiveFields($"{currentPath}", elementType!));
+                }
+            }
+        }
+
+        return list;
+    }
+
+    private static Type? GetElementType(PropertyInfo property)
+    {
+        if (property.PropertyType.IsArray)
+        {
+            return property.PropertyType.GetElementType()!;
+        }
+        else if (property.PropertyType.IsGenericType)
+        {
+            return property.PropertyType.GetGenericArguments()[0];
+
+        }
+        else if (property.PropertyType.IsClass)
+        {
+            return property.PropertyType;
+
+        }
+
+        return default;
+    }
+}