From b52a5423aeafb4184df2bf9bd27c64d859cfc8cd Mon Sep 17 00:00:00 2001 From: Martyn Whitwell Date: Thu, 15 Aug 2024 12:25:48 +0100 Subject: [PATCH 1/2] new endpoint for synchronising by candidate ID --- .../Controllers/OperationsController.cs | 71 ++++++++++++------- GetIntoTeachingApi/Jobs/ApplyBackfillJob.cs | 65 +++++++++++++++-- .../Models/CandidateIdsRequest.cs | 11 +++ .../Controllers/OperationsControllerTests.cs | 37 ++++++++++ .../Jobs/ApplyBackfillJobTests.cs | 38 ++++++++++ .../Models/CandidateIdsRequestTests.cs | 17 +++++ 6 files changed, 209 insertions(+), 30 deletions(-) create mode 100644 GetIntoTeachingApi/Models/CandidateIdsRequest.cs create mode 100644 GetIntoTeachingApiTests/Models/CandidateIdsRequestTests.cs diff --git a/GetIntoTeachingApi/Controllers/OperationsController.cs b/GetIntoTeachingApi/Controllers/OperationsController.cs index bde52c769..a7b7b5f58 100644 --- a/GetIntoTeachingApi/Controllers/OperationsController.cs +++ b/GetIntoTeachingApi/Controllers/OperationsController.cs @@ -2,13 +2,13 @@ using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; -using GetIntoTeachingApi.Jobs; +using GetIntoTeachingApi.Jobs; using GetIntoTeachingApi.Models; using GetIntoTeachingApi.Models.Crm; using GetIntoTeachingApi.Services; using GetIntoTeachingApi.Utils; -using Hangfire; -using Microsoft.AspNetCore.Authorization; +using Hangfire; +using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; using Swashbuckle.AspNetCore.Annotations; @@ -19,7 +19,7 @@ namespace GetIntoTeachingApi.Controllers [ApiController] public class OperationsController : ControllerBase { - // Must be a substantial amount longer than the max expected offline duration + // Must be a substantial amount longer than the max expected offline duration // of the CRM and less than 24 hours (the point at which we auto-purge any held PII). private static readonly TimeSpan CrmIntegrationAutoResumeInterval = TimeSpan.FromHours(6); private readonly IStore _store; @@ -28,7 +28,7 @@ public class OperationsController : ControllerBase private readonly IHangfireService _hangfire; private readonly IRedisService _redis; private readonly IEnv _env; - private readonly IAppSettings _appSettings; + private readonly IAppSettings _appSettings; private readonly IBackgroundJobClient _jobClient; public OperationsController( @@ -92,7 +92,7 @@ public async Task HealthCheck() return Ok(response); } - [HttpPut] + [HttpPut] [Authorize(Roles = "Admin,Crm")] [Route("pause_crm_integration")] [SwaggerOperation( @@ -108,14 +108,14 @@ public IActionResult PauseCrmIntegration() _appSettings.CrmIntegrationPausedUntil = DateTime.UtcNow.AddHours(CrmIntegrationAutoResumeInterval.TotalHours); return NoContent(); - } - - [HttpPut] + } + + [HttpPut] [Authorize(Roles = "Admin,Crm")] [Route("resume_crm_integration")] - [SwaggerOperation( - Summary = "Resumes the integration with the CRM (after being paused).", - OperationId = "ResumeCrmIntegration", + [SwaggerOperation( + Summary = "Resumes the integration with the CRM (after being paused).", + OperationId = "ResumeCrmIntegration", Tags = new[] { "Operations" })] [ProducesResponseType(typeof(HealthCheckResponse), StatusCodes.Status204NoContent)] public IActionResult ResumeCrmIntegration() @@ -123,27 +123,50 @@ public IActionResult ResumeCrmIntegration() _appSettings.CrmIntegrationPausedUntil = null; return NoContent(); - } - - [HttpPost] + } + + [HttpPost] [Authorize(Roles = "Admin")] [Route("backfill_apply_candidates")] - [SwaggerOperation( - Summary = "Triggers a backfill job to sync the CRM with the Apply candidates.", - Description = "The backfill will query all candidate information from the Apply API and " + - "queue jobs to sync the data with the CRM.", - OperationId = "BackfillApplyCandidates", + [SwaggerOperation( + Summary = "Triggers a backfill job to sync the CRM with the Apply candidates.", + Description = "The backfill will query all candidate information from the Apply API and " + + "queue jobs to sync the data with the CRM.", + OperationId = "BackfillApplyCandidates", Tags = new[] { "Operations" })] [ProducesResponseType(StatusCodes.Status204NoContent)] [ProducesResponseType(StatusCodes.Status400BadRequest)] public IActionResult BackfillApplyCandidates(DateTime updatedSince) { - if (_appSettings.IsApplyBackfillInProgress) - { - return BadRequest("Backfill already in progress"); + if (_appSettings.IsApplyBackfillInProgress) + { + return BadRequest("Backfill already in progress"); } + + _jobClient.Enqueue((x) => x.RunAsync(updatedSince, 1, null)); - _jobClient.Enqueue((x) => x.RunAsync(updatedSince, 1)); + return NoContent(); + } + + [HttpPost] + [Authorize(Roles = "Admin")] + [Route("backfill_apply_candidates_from_ids")] + [SwaggerOperation( + Summary = "Triggers a backfill job to sync the CRM with the Apply candidates for specified candidate Ids.", + Description = "The backfill will query all candidate information from the Apply API and " + + "queue jobs to sync the data with the CRM.", + OperationId = "BackfillApplyCandidatesFromIds", + Tags = new[] { "Operations" })] + [ProducesResponseType(StatusCodes.Status204NoContent)] + [ProducesResponseType(StatusCodes.Status400BadRequest)] + public IActionResult BackfillApplyCandidatesFromIds([FromBody, SwaggerRequestBody("Candidate IDs to backfill", Required = true)] CandidateIdsRequest request) + { + if (_appSettings.IsApplyBackfillInProgress) + { + return BadRequest("Backfill already in progress"); + } + + _jobClient.Enqueue((x) => x.RunAsync(DateTime.MinValue, 1, request.CandidateIds)); return NoContent(); } diff --git a/GetIntoTeachingApi/Jobs/ApplyBackfillJob.cs b/GetIntoTeachingApi/Jobs/ApplyBackfillJob.cs index a04986fbe..20a7cf829 100644 --- a/GetIntoTeachingApi/Jobs/ApplyBackfillJob.cs +++ b/GetIntoTeachingApi/Jobs/ApplyBackfillJob.cs @@ -1,6 +1,8 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Globalization; +using System.Threading; using System.Threading.Tasks; using Flurl; using Flurl.Http; @@ -12,8 +14,10 @@ using GetIntoTeachingApi.Utils; using Hangfire; using Hangfire.Server; +using Microsoft.AspNetCore.Connections; using Microsoft.Extensions.Logging; using MoreLinq; +using NuGet.Protocol; namespace GetIntoTeachingApi.Jobs { @@ -21,6 +25,7 @@ namespace GetIntoTeachingApi.Jobs public class ApplyBackfillJob : BaseJob { public static readonly int PagesPerJob = 10; + public static readonly int RecordsPerJob = 20; private readonly IBackgroundJobClient _jobClient; private readonly ILogger _logger; private readonly IAppSettings _appSettings; @@ -39,12 +44,19 @@ public ApplyBackfillJob( } [DisableConcurrentExecution(timeoutInSeconds: 60 * 60)] - public async Task RunAsync(DateTime updatedSince, int startPage = 1) + public async Task RunAsync(DateTime updatedSince, int startPage = 1, IEnumerable candidateIds = null) { _appSettings.IsApplyBackfillInProgress = true; - _logger.LogInformation("ApplyBackfillJob - Started - Pages {StartPage} to {EndPage}", startPage, EndPage(startPage)); - await QueueCandidateSyncJobs(updatedSince, startPage); - _logger.LogInformation("ApplyBackfillJob - Succeeded - Pages {StartPage} to {EndPage}", startPage, EndPage(startPage)); + _logger.LogInformation("ApplyBackfillJob - Started - Pages {StartPage} to {EndPage} (candidate IDs: {CandidateIds})", startPage, EndPage(startPage), candidateIds?.Any()); + + if (candidateIds?.Any() ?? false) + { + await QueueCandidateSyncJobsCandidateIds(candidateIds); + } else { + await QueueCandidateSyncJobsUpdatedSince(updatedSince, startPage); + } + + _logger.LogInformation("ApplyBackfillJob - Succeeded - Pages {StartPage} to {EndPage} (candidate IDs: {CandidateIds})", startPage, EndPage(startPage), candidateIds?.Any()); _appSettings.IsApplyBackfillInProgress = false; } @@ -53,7 +65,7 @@ private static int EndPage(int startPage) return startPage + PagesPerJob - 1; } - private async Task QueueCandidateSyncJobs(DateTime updatedSince, int startPage) + private async Task QueueCandidateSyncJobsUpdatedSince(DateTime updatedSince, int startPage) { // Enforce use of the Newtonsoft Json serializer FlurlHttp.Clients.UseNewtonsoft(); @@ -76,7 +88,48 @@ private async Task QueueCandidateSyncJobs(DateTime updatedSince, int startPage) // to process the next batch of pages. if (paginator.HasNext) { - _jobClient.Enqueue((x) => x.RunAsync(updatedSince, paginator.Page)); + _jobClient.Enqueue((x) => x.RunAsync(updatedSince, paginator.Page, null)); + } + } + + private async Task QueueCandidateSyncJobsCandidateIds(IEnumerable candidateIds) + { + // Enforce use of the Newtonsoft Json serializer + FlurlHttp.Clients.UseNewtonsoft(); + + var batch = candidateIds.Take(RecordsPerJob); + var remainder = candidateIds.Skip(RecordsPerJob); + + foreach (int candidateId in batch) + { + _logger.LogInformation("Fetching CandidateID C{CandidateId} from the Apply API", candidateId); + + try + { + var request = Env.ApplyCandidateApiUrl + .AppendPathSegment("candidates") + .AppendPathSegment(String.Format(CultureInfo.InvariantCulture, "C{0}", candidateId)) + .WithOAuthBearerToken(Env.ApplyCandidateApiKey); + + var candidate = await request.GetJsonAsync>(); + + _logger.LogInformation("Scheduling ApplyBackfillJob - Syncing CandidateID: C{Id}", candidateId); + + _jobClient.Schedule(x => x.Run(candidate.Data), TimeSpan.FromSeconds(60)); + } + catch (FlurlHttpException ex) + { + _logger.LogError("Failed to fetch CandidateID C{CandidateId} from the Apply API (status: {Status})", candidateId, ex.StatusCode); + } + + await Task.Delay(100); + } + + // When we reach the end page we re-queue the backfill job + // to process the next batch of candidate IDs. + if (remainder?.Any() ?? false) + { + _jobClient.Enqueue((x) => x.RunAsync(DateTime.MinValue, 1, remainder.ToArray())); } } } diff --git a/GetIntoTeachingApi/Models/CandidateIdsRequest.cs b/GetIntoTeachingApi/Models/CandidateIdsRequest.cs new file mode 100644 index 000000000..252cbbc60 --- /dev/null +++ b/GetIntoTeachingApi/Models/CandidateIdsRequest.cs @@ -0,0 +1,11 @@ +using System; +using System.Globalization; +using System.Linq; + +namespace GetIntoTeachingApi.Models +{ + public class CandidateIdsRequest + { + public int[] CandidateIds { get; set; } + } +} diff --git a/GetIntoTeachingApiTests/Controllers/OperationsControllerTests.cs b/GetIntoTeachingApiTests/Controllers/OperationsControllerTests.cs index 0bb9a25e5..c4d23dda6 100644 --- a/GetIntoTeachingApiTests/Controllers/OperationsControllerTests.cs +++ b/GetIntoTeachingApiTests/Controllers/OperationsControllerTests.cs @@ -138,6 +138,43 @@ public void BackfillApplyCandidates_WhenAlreadyRunning_ReturnsBadRequest() It.Is(job => job.Type == typeof(ApplyBackfillJob) && job.Method.Name == "RunAsync" && (DateTime)job.Args[0] == updatedSince), It.IsAny()),Times.Never); } + + [Fact] + public void BackfillApplyCandidatesFromIds_Authorize_IsPresent() + { + typeof(OperationsController).GetMethod("BackfillApplyCandidatesFromIds") + .Should().BeDecoratedWith(a => a.Roles == "Admin"); + } + + [Fact] + public void BackfillApplyCandidatesFromIds_WhenNotAlreadyRunning_EnqueuesJob() + { + _mockAppSettings.Setup(m => m.IsApplyBackfillInProgress).Returns(false); + var candidateIds = new[] { 1, 2, 3 }; + var request = new CandidateIdsRequest { CandidateIds = candidateIds }; + var response = _controller.BackfillApplyCandidatesFromIds(request); + + response.Should().BeOfType(); + + _mockJobClient.Verify(x => x.Create( + It.Is(job => job.Type == typeof(ApplyBackfillJob) && job.Method.Name == "RunAsync" && (int[])job.Args[2] == candidateIds), + It.IsAny()), Times.Once); + } + + [Fact] + public void BackfillApplyCandidatesFromIds_WhenAlreadyRunning_ReturnsBadRequest() + { + _mockAppSettings.Setup(m => m.IsApplyBackfillInProgress).Returns(true); + var candidateIds = new[] { 1, 2, 3 }; + var request = new CandidateIdsRequest { CandidateIds = candidateIds }; + var response = _controller.BackfillApplyCandidatesFromIds(request); + + response.Should().BeOfType(); + + _mockJobClient.Verify(x => x.Create( + It.Is(job => job.Type == typeof(ApplyBackfillJob) && job.Method.Name == "RunAsync" && (int[])job.Args[2] == candidateIds), + It.IsAny()),Times.Never); + } [Theory] [InlineData(true, true, true, true, true, "healthy")] diff --git a/GetIntoTeachingApiTests/Jobs/ApplyBackfillJobTests.cs b/GetIntoTeachingApiTests/Jobs/ApplyBackfillJobTests.cs index 4f47be0d1..d1051849a 100644 --- a/GetIntoTeachingApiTests/Jobs/ApplyBackfillJobTests.cs +++ b/GetIntoTeachingApiTests/Jobs/ApplyBackfillJobTests.cs @@ -86,6 +86,33 @@ public async Task RunAsync_WhenMultiplePagesAvailable_QueuesCandidateJobsForEach (DateTime)job.Args[0] == updatedSince && (int)job.Args[1] == (ApplyBackfillJob.PagesPerJob + 1)), It.IsAny()), Times.Once); } + + [Fact] + public async Task RunAsync_WithCandidateIds() + { + int[] candidateIds = new[] { 1,2,3 }; + var candidate1 = new Candidate() { Id = "1", Attributes = new CandidateAttributes() { Email = $"email1@address.com" } }; + var candidate2 = new Candidate() { Id = "2", Attributes = new CandidateAttributes() { Email = $"email2@address.com" } }; + var candidate3 = new Candidate() { Id = "3", Attributes = new CandidateAttributes() { Email = $"email3@address.com" } }; + + using (var httpTest = new HttpTest()) + { + MockIndividualResponse(httpTest, new Response() { Data = candidate1 }, 1); + MockIndividualResponse(httpTest, new Response() { Data = candidate2 }, 2); + MockIndividualResponse(httpTest, new Response() { Data = candidate3 }, 3); + + await _job.RunAsync(DateTime.MinValue, 1, candidateIds); + } + + _mockJobClient.Verify(x => x.Create( + It.Is(job => job.Type == typeof(ApplyCandidateSyncJob) && job.Method.Name == "Run"), + It.IsAny()), Times.Exactly(3)); + + _mockAppSettings.VerifySet(m => m.IsApplyBackfillInProgress = true, Times.Once); + _mockLogger.VerifyInformationWasCalled("ApplyBackfillJob - Started - Pages 1 to 10"); + _mockLogger.VerifyInformationWasCalled("ApplyBackfillJob - Succeeded - Pages 1 to 10"); + _mockAppSettings.VerifySet(m => m.IsApplyBackfillInProgress = false, Times.Once); + } [Fact] public async Task RunAsync_WithNoCandidates_DoesNotQueueJobs() @@ -122,5 +149,16 @@ private void MockResponse(HttpTest httpTest, Response> re .WithHeader("Authorization", $"Bearer {_mockEnv.Object.ApplyCandidateApiKey}") .RespondWith(json, 200, headers); } + + private void MockIndividualResponse(HttpTest httpTest, Response response, int candidateId) + { + var json = JsonConvert.SerializeObject(response); + + httpTest + .ForCallsTo($"{_mockEnv.Object.ApplyCandidateApiUrl}/candidates/C{candidateId}") + .WithVerb("GET") + .WithHeader("Authorization", $"Bearer {_mockEnv.Object.ApplyCandidateApiKey}") + .RespondWith(json, 200); + } } } \ No newline at end of file diff --git a/GetIntoTeachingApiTests/Models/CandidateIdsRequestTests.cs b/GetIntoTeachingApiTests/Models/CandidateIdsRequestTests.cs new file mode 100644 index 000000000..15c454c76 --- /dev/null +++ b/GetIntoTeachingApiTests/Models/CandidateIdsRequestTests.cs @@ -0,0 +1,17 @@ +using FluentAssertions; +using GetIntoTeachingApi.Models; +using Xunit; + +namespace GetIntoTeachingApiTests.Models +{ + public class CandidateIdsRequestTests + { + [Fact] + public void Constructor_WithInitializer() + { + var candidateIdsRequest = new CandidateIdsRequest { CandidateIds = new[] { 1, 2, 3 } }; + + candidateIdsRequest.CandidateIds.Should().Equal(1, 2, 3); + } + } +} From bf33e18e024992cb862ad95b5c710953414fbe0c Mon Sep 17 00:00:00 2001 From: Martyn Whitwell Date: Thu, 15 Aug 2024 16:13:37 +0100 Subject: [PATCH 2/2] Update ApplyBackfillJob.cs --- GetIntoTeachingApi/Jobs/ApplyBackfillJob.cs | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/GetIntoTeachingApi/Jobs/ApplyBackfillJob.cs b/GetIntoTeachingApi/Jobs/ApplyBackfillJob.cs index 20a7cf829..c4e2c9a8a 100644 --- a/GetIntoTeachingApi/Jobs/ApplyBackfillJob.cs +++ b/GetIntoTeachingApi/Jobs/ApplyBackfillJob.cs @@ -48,13 +48,10 @@ public async Task RunAsync(DateTime updatedSince, int startPage = 1, IEnumerable { _appSettings.IsApplyBackfillInProgress = true; _logger.LogInformation("ApplyBackfillJob - Started - Pages {StartPage} to {EndPage} (candidate IDs: {CandidateIds})", startPage, EndPage(startPage), candidateIds?.Any()); - - if (candidateIds?.Any() ?? false) - { - await QueueCandidateSyncJobsCandidateIds(candidateIds); - } else { - await QueueCandidateSyncJobsUpdatedSince(updatedSince, startPage); - } + + await (candidateIds?.Any() ?? false + ? QueueCandidateSyncJobsCandidateIds(candidateIds) + : QueueCandidateSyncJobsUpdatedSince(updatedSince, startPage)); _logger.LogInformation("ApplyBackfillJob - Succeeded - Pages {StartPage} to {EndPage} (candidate IDs: {CandidateIds})", startPage, EndPage(startPage), candidateIds?.Any()); _appSettings.IsApplyBackfillInProgress = false; @@ -122,7 +119,7 @@ private async Task QueueCandidateSyncJobsCandidateIds(IEnumerable candidate _logger.LogError("Failed to fetch CandidateID C{CandidateId} from the Apply API (status: {Status})", candidateId, ex.StatusCode); } - await Task.Delay(100); + await Task.Delay(100); // add a short delay so as not to overwhelm the Apply API } // When we reach the end page we re-queue the backfill job