Skip to content

Commit

Permalink
updated to download everything within a single bundle
Browse files Browse the repository at this point in the history
  • Loading branch information
Thomas Anderson committed Dec 3, 2024
1 parent 99e2413 commit 4ce11e7
Showing 4 changed files with 63 additions and 26 deletions.
52 changes: 35 additions & 17 deletions Cdms.Business/Commands/DownloadNotificationsCommand.cs
Original file line number Diff line number Diff line change
@@ -12,48 +12,66 @@
using Cdms.SensitiveData;
using Cdms.Types.Ipaffs;
using Microsoft.AspNetCore.Hosting;
using Cdms.SyncJob;
using Cdms.Types.Alvs;
using Cdms.Types.Gvms;

namespace Cdms.Business.Commands;

public class DownloadCommand : IRequest<DownloadCommand.Result>
public class DownloadCommand : IRequest, ISyncJob
{
[System.Text.Json.Serialization.JsonConverter(typeof(JsonStringEnumConverter<SyncPeriod>))]
public SyncPeriod SyncPeriod { get; set; }

public string Path { get; set; } = null!;
public Guid JobId { get; } = Guid.NewGuid();
public string Timespan { get; } = null!;
public string Resource { get; } = null!;

public Type Type { get; set; } = null!;

internal class Handler(IBlobService blobService, ISensitiveDataSerializer sensitiveDataSerializer, IWebHostEnvironment env) : IRequestHandler<DownloadCommand, DownloadCommand.Result>
internal class Handler(IBlobService blobService, ISensitiveDataSerializer sensitiveDataSerializer, IWebHostEnvironment env) : IRequestHandler<DownloadCommand>
{

public async Task<Result> Handle(DownloadCommand request, CancellationToken cancellationToken)
public async Task Handle(DownloadCommand request, CancellationToken cancellationToken)
{
string subFolder = $"{request.Type.Name}\\{Guid.NewGuid()}";
string rootFolder = System.IO.Path.Combine(env.ContentRootPath, subFolder);
string subFolder = $"temp\\{request.JobId}";
string rootFolder = Path.Combine(env.ContentRootPath, subFolder);
Directory.CreateDirectory(rootFolder);

await Download(request, rootFolder, "RAW/IPAFFS/CHEDA", typeof(ImportNotification), cancellationToken);
await Download(request, rootFolder, "RAW/IPAFFS/CHEDD", typeof(ImportNotification), cancellationToken);
await Download(request, rootFolder, "RAW/IPAFFS/CHEDP", typeof(ImportNotification), cancellationToken);
await Download(request, rootFolder, "RAW/IPAFFS/CHEDPP", typeof(ImportNotification), cancellationToken);

await Download(request, rootFolder, "RAW/ALVS", typeof(AlvsClearanceRequest), cancellationToken);

await Download(request, rootFolder, "RAW/GVMSAPIRESPONSE", typeof(SearchGmrsForDeclarationIdsResponse), cancellationToken);

await Download(request, rootFolder, "RAW/DECISIONS", typeof(AlvsClearanceRequest), cancellationToken);

ZipFile.CreateFromDirectory(rootFolder, $"{env.ContentRootPath}\\{request.JobId}.zip");

Directory.Delete(rootFolder, true);
}

private async Task Download(DownloadCommand request, string rootFolder, string folder, Type type, CancellationToken cancellationToken)
{

ParallelOptions options = new() { CancellationToken = cancellationToken, MaxDegreeOfParallelism = 10 };
var result = blobService.GetResourcesAsync($"{request.Path}{request.SyncPeriod.GetPeriodPath()}", cancellationToken);
var result = blobService.GetResourcesAsync($"{folder}{request.SyncPeriod.GetPeriodPath()}", cancellationToken);

//Write local files
await Parallel.ForEachAsync(result, options, async (item, token) =>
{
var blobContent = await blobService.GetResource(item, cancellationToken);
string redactedContent = sensitiveDataSerializer.RedactRawJson(blobContent, request.Type);
string redactedContent = sensitiveDataSerializer.RedactRawJson(blobContent, type);
var filename = System.IO.Path.Combine(rootFolder, item.Name.Replace('/', System.IO.Path.DirectorySeparatorChar));
Directory.CreateDirectory(System.IO.Path.GetDirectoryName(filename)!);
await File.WriteAllTextAsync(filename, redactedContent, cancellationToken);
});

MemoryStream zipStream = new MemoryStream();
ZipFile.CreateFromDirectory(rootFolder, zipStream);
zipStream.Position = 0;
var commandResult = new Result(zipStream.ToArray());
Directory.Delete(rootFolder, true);
return commandResult;
}

}

public record Result(byte[] Zip);


}
18 changes: 10 additions & 8 deletions CdmsBackend/Endpoints/SyncEndpoints.cs
Original file line number Diff line number Diff line change
@@ -11,6 +11,7 @@
using MediatR;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Options;
using static Cdms.Business.Commands.DownloadCommand;

namespace CdmsBackend.Endpoints;

@@ -27,8 +28,8 @@ public static void UseSyncEndpoints(this IEndpointRouteBuilder app, IOptions<Api
app.MapGet(BaseRoute + "/clearance-requests/", GetSyncClearanceRequests).AllowAnonymous();
app.MapPost(BaseRoute + "/clearance-requests/", SyncClearanceRequests).AllowAnonymous();

app.MapGet(BaseRoute + "/download/import-notifications", DownloadNotifications).AllowAnonymous();
app.MapGet(BaseRoute + "/download/clearance-requests", DownloadClearanceRequests).AllowAnonymous();
app.MapGet(BaseRoute + "/generate-download", GenerateDownload).AllowAnonymous();
app.MapGet(BaseRoute + "/download/{id}", DownloadNotifications).AllowAnonymous();
}

app.MapGet(BaseRoute + "/gmrs/", GetSyncGmrs).AllowAnonymous();
@@ -55,16 +56,17 @@ internal static async Task<IResult> InitialiseEnvironment(IHost app, SyncPeriod
return Results.Ok();
}

private static async Task<IResult> DownloadNotifications([FromServices] IMediator mediator, [FromQuery] string path, [FromQuery]SyncPeriod period)
private static IResult DownloadNotifications([FromServices] IWebHostEnvironment env, string id)
{
var result = await mediator.Send(new DownloadCommand() { Type = typeof(ImportNotification), Path = path, SyncPeriod = period });
return Results.File(result.Zip, "application/zip", "notifications.zip");
var stream = File.OpenRead($"{System.IO.Path.Combine(env.ContentRootPath, id)}.zip");
return Results.File(stream, "application/zip", $"{id}.zip");
}

private static async Task<IResult> DownloadClearanceRequests([FromServices] IMediator mediator, [FromQuery] string path, [FromQuery] SyncPeriod period)
private static async Task<IResult> GenerateDownload([FromServices] ICdmsMediator mediator, [FromQuery] SyncPeriod period)
{
var result = await mediator.Send(new DownloadCommand() { Type = typeof(AlvsClearanceRequest), Path = path, SyncPeriod = period });
return Results.File(result.Zip, "application/zip", "clearancerequests.zip");
var command = new DownloadCommand() { SyncPeriod = period };
await mediator.SendJob(command);
return Results.Ok(command.JobId);
}

private static Task<IResult> GetAllSyncJobs([FromServices] ISyncJobStore store)
14 changes: 14 additions & 0 deletions CdmsBackend/Mediatr/CdmsMediator.cs
Original file line number Diff line number Diff line change
@@ -30,6 +30,20 @@ await backgroundTaskQueue.QueueBackgroundWorkItemAsync(async (ct) =>
});
}

public async Task SendJob<TRequest>(TRequest request, CancellationToken cancellationToken = default) where TRequest : IRequest, ISyncJob
{
var job = syncJobStore.CreateJob(request.JobId, request.Timespan, request.Resource);

await backgroundTaskQueue.QueueBackgroundWorkItemAsync(async (ct) =>
{
using var scope = serviceScopeFactory.CreateScope();
using var activity = ActivitySource.StartActivity(ActivityName, ActivityKind.Client);
var m = scope.ServiceProvider.GetRequiredService<IMediator>();
await m.Send(request, job.CancellationToken);
job.Complete();
});
}

Task<TResponse> ICdmsMediator.Send<TResponse>(IRequest<TResponse> request, CancellationToken cancellationToken)
{
return mediator.Send(request, cancellationToken);
5 changes: 4 additions & 1 deletion CdmsBackend/Mediatr/ICdmsMediator.cs
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
using Cdms.SyncJob;
using Cdms.SyncJob;
using MediatR;

namespace CdmsBackend.Mediatr;
@@ -8,6 +8,9 @@ public interface ICdmsMediator
Task SendSyncJob<TRequest>(TRequest request, CancellationToken cancellationToken = default)
where TRequest : IRequest, ISyncJob;

Task SendJob<TRequest>(TRequest request, CancellationToken cancellationToken = default)
where TRequest : IRequest, ISyncJob;



/// <summary>

0 comments on commit 4ce11e7

Please sign in to comment.