diff --git a/Cdms.Business/Commands/DownloadNotificationsCommand.cs b/Cdms.Business/Commands/DownloadNotificationsCommand.cs index 453599b3..be241fde 100644 --- a/Cdms.Business/Commands/DownloadNotificationsCommand.cs +++ b/Cdms.Business/Commands/DownloadNotificationsCommand.cs @@ -12,48 +12,66 @@ using Cdms.SensitiveData; using Cdms.Types.Ipaffs; using Microsoft.AspNetCore.Hosting; +using Cdms.SyncJob; +using Cdms.Types.Alvs; +using Cdms.Types.Gvms; namespace Cdms.Business.Commands; -public class DownloadCommand : IRequest +public class DownloadCommand : IRequest, ISyncJob { [System.Text.Json.Serialization.JsonConverter(typeof(JsonStringEnumConverter))] public SyncPeriod SyncPeriod { get; set; } - public string Path { get; set; } = null!; + public Guid JobId { get; } = Guid.NewGuid(); + public string Timespan { get; } = null!; + public string Resource { get; } = null!; - public Type Type { get; set; } = null!; - - internal class Handler(IBlobService blobService, ISensitiveDataSerializer sensitiveDataSerializer, IWebHostEnvironment env) : IRequestHandler + internal class Handler(IBlobService blobService, ISensitiveDataSerializer sensitiveDataSerializer, IWebHostEnvironment env) : IRequestHandler { - public async Task Handle(DownloadCommand request, CancellationToken cancellationToken) + public async Task Handle(DownloadCommand request, CancellationToken cancellationToken) { - string subFolder = $"{request.Type.Name}\\{Guid.NewGuid()}"; - string rootFolder = System.IO.Path.Combine(env.ContentRootPath, subFolder); + string subFolder = $"temp\\{request.JobId}"; + string rootFolder = Path.Combine(env.ContentRootPath, subFolder); Directory.CreateDirectory(rootFolder); + + await Download(request, rootFolder, "RAW/IPAFFS/CHEDA", typeof(ImportNotification), cancellationToken); + await Download(request, rootFolder, "RAW/IPAFFS/CHEDD", typeof(ImportNotification), cancellationToken); + await Download(request, rootFolder, "RAW/IPAFFS/CHEDP", typeof(ImportNotification), cancellationToken); + await Download(request, rootFolder, "RAW/IPAFFS/CHEDPP", typeof(ImportNotification), cancellationToken); + + await Download(request, rootFolder, "RAW/ALVS", typeof(AlvsClearanceRequest), cancellationToken); + + await Download(request, rootFolder, "RAW/GVMSAPIRESPONSE", typeof(SearchGmrsForDeclarationIdsResponse), cancellationToken); + + await Download(request, rootFolder, "RAW/DECISIONS", typeof(AlvsClearanceRequest), cancellationToken); + + ZipFile.CreateFromDirectory(rootFolder, $"{env.ContentRootPath}\\{request.JobId}.zip"); + + Directory.Delete(rootFolder, true); + } + + private async Task Download(DownloadCommand request, string rootFolder, string folder, Type type, CancellationToken cancellationToken) + { + ParallelOptions options = new() { CancellationToken = cancellationToken, MaxDegreeOfParallelism = 10 }; - var result = blobService.GetResourcesAsync($"{request.Path}{request.SyncPeriod.GetPeriodPath()}", cancellationToken); + var result = blobService.GetResourcesAsync($"{folder}{request.SyncPeriod.GetPeriodPath()}", cancellationToken); //Write local files await Parallel.ForEachAsync(result, options, async (item, token) => { var blobContent = await blobService.GetResource(item, cancellationToken); - string redactedContent = sensitiveDataSerializer.RedactRawJson(blobContent, request.Type); + string redactedContent = sensitiveDataSerializer.RedactRawJson(blobContent, type); var filename = System.IO.Path.Combine(rootFolder, item.Name.Replace('/', System.IO.Path.DirectorySeparatorChar)); Directory.CreateDirectory(System.IO.Path.GetDirectoryName(filename)!); await File.WriteAllTextAsync(filename, redactedContent, cancellationToken); }); - - MemoryStream zipStream = new MemoryStream(); - ZipFile.CreateFromDirectory(rootFolder, zipStream); - zipStream.Position = 0; - var commandResult = new Result(zipStream.ToArray()); - Directory.Delete(rootFolder, true); - return commandResult; } } public record Result(byte[] Zip); + + } \ No newline at end of file diff --git a/CdmsBackend/Endpoints/SyncEndpoints.cs b/CdmsBackend/Endpoints/SyncEndpoints.cs index a79c9a06..b5f51b1e 100644 --- a/CdmsBackend/Endpoints/SyncEndpoints.cs +++ b/CdmsBackend/Endpoints/SyncEndpoints.cs @@ -11,6 +11,7 @@ using MediatR; using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Options; +using static Cdms.Business.Commands.DownloadCommand; namespace CdmsBackend.Endpoints; @@ -27,8 +28,8 @@ public static void UseSyncEndpoints(this IEndpointRouteBuilder app, IOptions InitialiseEnvironment(IHost app, SyncPeriod return Results.Ok(); } - private static async Task DownloadNotifications([FromServices] IMediator mediator, [FromQuery] string path, [FromQuery]SyncPeriod period) + private static IResult DownloadNotifications([FromServices] IWebHostEnvironment env, string id) { - var result = await mediator.Send(new DownloadCommand() { Type = typeof(ImportNotification), Path = path, SyncPeriod = period }); - return Results.File(result.Zip, "application/zip", "notifications.zip"); + var stream = File.OpenRead($"{System.IO.Path.Combine(env.ContentRootPath, id)}.zip"); + return Results.File(stream, "application/zip", $"{id}.zip"); } - private static async Task DownloadClearanceRequests([FromServices] IMediator mediator, [FromQuery] string path, [FromQuery] SyncPeriod period) + private static async Task GenerateDownload([FromServices] ICdmsMediator mediator, [FromQuery] SyncPeriod period) { - var result = await mediator.Send(new DownloadCommand() { Type = typeof(AlvsClearanceRequest), Path = path, SyncPeriod = period }); - return Results.File(result.Zip, "application/zip", "clearancerequests.zip"); + var command = new DownloadCommand() { SyncPeriod = period }; + await mediator.SendJob(command); + return Results.Ok(command.JobId); } private static Task GetAllSyncJobs([FromServices] ISyncJobStore store) diff --git a/CdmsBackend/Mediatr/CdmsMediator.cs b/CdmsBackend/Mediatr/CdmsMediator.cs index 213411d3..56d5c425 100644 --- a/CdmsBackend/Mediatr/CdmsMediator.cs +++ b/CdmsBackend/Mediatr/CdmsMediator.cs @@ -30,6 +30,20 @@ await backgroundTaskQueue.QueueBackgroundWorkItemAsync(async (ct) => }); } + public async Task SendJob(TRequest request, CancellationToken cancellationToken = default) where TRequest : IRequest, ISyncJob + { + var job = syncJobStore.CreateJob(request.JobId, request.Timespan, request.Resource); + + await backgroundTaskQueue.QueueBackgroundWorkItemAsync(async (ct) => + { + using var scope = serviceScopeFactory.CreateScope(); + using var activity = ActivitySource.StartActivity(ActivityName, ActivityKind.Client); + var m = scope.ServiceProvider.GetRequiredService(); + await m.Send(request, job.CancellationToken); + job.Complete(); + }); + } + Task ICdmsMediator.Send(IRequest request, CancellationToken cancellationToken) { return mediator.Send(request, cancellationToken); diff --git a/CdmsBackend/Mediatr/ICdmsMediator.cs b/CdmsBackend/Mediatr/ICdmsMediator.cs index fd4b5ea3..6443f1b3 100644 --- a/CdmsBackend/Mediatr/ICdmsMediator.cs +++ b/CdmsBackend/Mediatr/ICdmsMediator.cs @@ -1,4 +1,4 @@ -using Cdms.SyncJob; +using Cdms.SyncJob; using MediatR; namespace CdmsBackend.Mediatr; @@ -8,6 +8,9 @@ public interface ICdmsMediator Task SendSyncJob(TRequest request, CancellationToken cancellationToken = default) where TRequest : IRequest, ISyncJob; + Task SendJob(TRequest request, CancellationToken cancellationToken = default) + where TRequest : IRequest, ISyncJob; + ///