New endpoint for synchronising by candidate ID

GetIntoTeachingApi/Controllers/OperationsController.cs

@@ -2,13 +2,13 @@
 using System.Collections.Generic;
 using System.Linq;
 using System.Threading.Tasks;
-using GetIntoTeachingApi.Jobs;
+using GetIntoTeachingApi.Jobs;
 using GetIntoTeachingApi.Models;
 using GetIntoTeachingApi.Models.Crm;
 using GetIntoTeachingApi.Services;
 using GetIntoTeachingApi.Utils;
-using Hangfire;
-using Microsoft.AspNetCore.Authorization;
+using Hangfire;
+using Microsoft.AspNetCore.Authorization;
 using Microsoft.AspNetCore.Http;
 using Microsoft.AspNetCore.Mvc;
 using Swashbuckle.AspNetCore.Annotations;
@@ -19,7 +19,7 @@ namespace GetIntoTeachingApi.Controllers
     [ApiController]
     public class OperationsController : ControllerBase
     {
-        // Must be a substantial amount longer than the max expected offline duration
+        // Must be a substantial amount longer than the max expected offline duration
         // of the CRM and less than 24 hours (the point at which we auto-purge any held PII).
         private static readonly TimeSpan CrmIntegrationAutoResumeInterval = TimeSpan.FromHours(6);
         private readonly IStore _store;
@@ -28,7 +28,7 @@ public class OperationsController : ControllerBase
         private readonly IHangfireService _hangfire;
         private readonly IRedisService _redis;
         private readonly IEnv _env;
-        private readonly IAppSettings _appSettings;
+        private readonly IAppSettings _appSettings;
         private readonly IBackgroundJobClient _jobClient;
 
         public OperationsController(
@@ -92,7 +92,7 @@ public async Task<IActionResult> HealthCheck()
             return Ok(response);
         }
 
-        [HttpPut]
+        [HttpPut]
         [Authorize(Roles = "Admin,Crm")]
         [Route("pause_crm_integration")]
         [SwaggerOperation(
@@ -108,42 +108,65 @@ public IActionResult PauseCrmIntegration()
             _appSettings.CrmIntegrationPausedUntil = DateTime.UtcNow.AddHours(CrmIntegrationAutoResumeInterval.TotalHours);
 
             return NoContent();
-        }
-
-        [HttpPut]
+        }
+
+        [HttpPut]
         [Authorize(Roles = "Admin,Crm")]
         [Route("resume_crm_integration")]
-        [SwaggerOperation(
-            Summary = "Resumes the integration with the CRM (after being paused).",
-            OperationId = "ResumeCrmIntegration",
+        [SwaggerOperation(
+            Summary = "Resumes the integration with the CRM (after being paused).",
+            OperationId = "ResumeCrmIntegration",
             Tags = new[] { "Operations" })]
         [ProducesResponseType(typeof(HealthCheckResponse), StatusCodes.Status204NoContent)]
         public IActionResult ResumeCrmIntegration()
         {
             _appSettings.CrmIntegrationPausedUntil = null;
 
             return NoContent();
-        }
-
-        [HttpPost]
+        }
+
+        [HttpPost]
         [Authorize(Roles = "Admin")]
         [Route("backfill_apply_candidates")]
-        [SwaggerOperation(
-            Summary = "Triggers a backfill job to sync the CRM with the Apply candidates.",
-            Description = "The backfill will query all candidate information from the Apply API and " +
-            "queue jobs to sync the data with the CRM.",
-            OperationId = "BackfillApplyCandidates",
+        [SwaggerOperation(
+            Summary = "Triggers a backfill job to sync the CRM with the Apply candidates.",
+            Description = "The backfill will query all candidate information from the Apply API and " +
+            "queue jobs to sync the data with the CRM.",
+            OperationId = "BackfillApplyCandidates",
             Tags = new[] { "Operations" })]
         [ProducesResponseType(StatusCodes.Status204NoContent)]
         [ProducesResponseType(StatusCodes.Status400BadRequest)]
         public IActionResult BackfillApplyCandidates(DateTime updatedSince)
         {
-            if (_appSettings.IsApplyBackfillInProgress)
-            {
-                return BadRequest("Backfill already in progress");
+            if (_appSettings.IsApplyBackfillInProgress)
+            {
+                return BadRequest("Backfill already in progress");
             }
+
+            _jobClient.Enqueue<ApplyBackfillJob>((x) => x.RunAsync(updatedSince, 1, null));
 
-            _jobClient.Enqueue<ApplyBackfillJob>((x) => x.RunAsync(updatedSince, 1));
+            return NoContent();
+        }
+
+        [HttpPost]
+        [Authorize(Roles = "Admin")]
+        [Route("backfill_apply_candidates_from_ids")]
+        [SwaggerOperation(
+            Summary = "Triggers a backfill job to sync the CRM with the Apply candidates for specified candidate Ids.",
+            Description = "The backfill will query all candidate information from the Apply API and " +
+                          "queue jobs to sync the data with the CRM.",
+            OperationId = "BackfillApplyCandidatesFromIds",
+            Tags = new[] { "Operations" })]
+        [ProducesResponseType(StatusCodes.Status204NoContent)]
+        [ProducesResponseType(StatusCodes.Status400BadRequest)]
+        public IActionResult BackfillApplyCandidatesFromIds([FromBody, SwaggerRequestBody("Candidate IDs to backfill", Required = true)] CandidateIdsRequest request)
+        {
+            if (_appSettings.IsApplyBackfillInProgress)
+            {
+                return BadRequest("Backfill already in progress");
+            }
+
+            _jobClient.Enqueue<ApplyBackfillJob>((x) => x.RunAsync(DateTime.MinValue, 1, request.CandidateIds));
 
             return NoContent();
         }

GetIntoTeachingApi/Jobs/ApplyBackfillJob.cs

@@ -1,6 +1,8 @@
 using System;
 using System.Collections.Generic;
 using System.Linq;
+using System.Globalization;
+using System.Threading;
 using System.Threading.Tasks;
 using Flurl;
 using Flurl.Http;
@@ -12,15 +14,18 @@
 using GetIntoTeachingApi.Utils;
 using Hangfire;
 using Hangfire.Server;
+using Microsoft.AspNetCore.Connections;
 using Microsoft.Extensions.Logging;
 using MoreLinq;
+using NuGet.Protocol;
 
 namespace GetIntoTeachingApi.Jobs
 {
     [AutomaticRetry(Attempts = 0)]
     public class ApplyBackfillJob : BaseJob
     {
         public static readonly int PagesPerJob = 10;
+        public static readonly int RecordsPerJob = 20;
         private readonly IBackgroundJobClient _jobClient;
         private readonly ILogger<ApplyBackfillJob> _logger;
         private readonly IAppSettings _appSettings;
@@ -39,12 +44,16 @@ public ApplyBackfillJob(
         }
 
         [DisableConcurrentExecution(timeoutInSeconds: 60 * 60)]
-        public async Task RunAsync(DateTime updatedSince, int startPage = 1)
+        public async Task RunAsync(DateTime updatedSince, int startPage = 1, IEnumerable<int> candidateIds = null)
         {
             _appSettings.IsApplyBackfillInProgress = true;
-            _logger.LogInformation("ApplyBackfillJob - Started - Pages {StartPage} to {EndPage}", startPage, EndPage(startPage));
-            await QueueCandidateSyncJobs(updatedSince, startPage);
-            _logger.LogInformation("ApplyBackfillJob - Succeeded - Pages {StartPage} to {EndPage}", startPage, EndPage(startPage));
+            _logger.LogInformation("ApplyBackfillJob - Started - Pages {StartPage} to {EndPage} (candidate IDs: {CandidateIds})", startPage, EndPage(startPage), candidateIds?.Any());
+
+            await (candidateIds?.Any() ?? false
+                ? QueueCandidateSyncJobsCandidateIds(candidateIds)
+                : QueueCandidateSyncJobsUpdatedSince(updatedSince, startPage));
+
+            _logger.LogInformation("ApplyBackfillJob - Succeeded - Pages {StartPage} to {EndPage} (candidate IDs: {CandidateIds})", startPage, EndPage(startPage), candidateIds?.Any());
             _appSettings.IsApplyBackfillInProgress = false;
         }
 
@@ -53,7 +62,7 @@ private static int EndPage(int startPage)
             return startPage + PagesPerJob - 1;
         }
 
-        private async Task QueueCandidateSyncJobs(DateTime updatedSince, int startPage)
+        private async Task QueueCandidateSyncJobsUpdatedSince(DateTime updatedSince, int startPage)
         {
             // Enforce use of the Newtonsoft Json serializer
             FlurlHttp.Clients.UseNewtonsoft();
@@ -76,7 +85,48 @@ private async Task QueueCandidateSyncJobs(DateTime updatedSince, int startPage)
             // to process the next batch of pages.
             if (paginator.HasNext)
             {
-                _jobClient.Enqueue<ApplyBackfillJob>((x) => x.RunAsync(updatedSince, paginator.Page));
+                _jobClient.Enqueue<ApplyBackfillJob>((x) => x.RunAsync(updatedSince, paginator.Page, null));
+            }
+        }
+
+        private async Task QueueCandidateSyncJobsCandidateIds(IEnumerable<int> candidateIds)
+        {
+            // Enforce use of the Newtonsoft Json serializer
+            FlurlHttp.Clients.UseNewtonsoft();
+
+            var batch = candidateIds.Take(RecordsPerJob);
+            var remainder = candidateIds.Skip(RecordsPerJob);
+
+            foreach (int candidateId in batch) 
+            {
+                _logger.LogInformation("Fetching CandidateID C{CandidateId} from the Apply API", candidateId);
+
+                try
+                {
+                    var request = Env.ApplyCandidateApiUrl
+                        .AppendPathSegment("candidates")
+                        .AppendPathSegment(String.Format(CultureInfo.InvariantCulture, "C{0}", candidateId))
+                        .WithOAuthBearerToken(Env.ApplyCandidateApiKey);
+
+                    var candidate = await request.GetJsonAsync<Response<GetIntoTeachingApi.Models.Apply.Candidate>>();
+
+                    _logger.LogInformation("Scheduling ApplyBackfillJob - Syncing CandidateID: C{Id}", candidateId);
+
+                    _jobClient.Schedule<ApplyCandidateSyncJob>(x => x.Run(candidate.Data), TimeSpan.FromSeconds(60));
+                }
+                catch (FlurlHttpException ex)
+                {
+                    _logger.LogError("Failed to fetch CandidateID C{CandidateId} from the Apply API (status: {Status})", candidateId, ex.StatusCode);
+                }
+
+                await Task.Delay(100); // add a short delay so as not to overwhelm the Apply API
+            }
+
+            // When we reach the end page we re-queue the backfill job
+            // to process the next batch of candidate IDs.
+            if (remainder?.Any() ?? false)
+            {
+                _jobClient.Enqueue<ApplyBackfillJob>((x) => x.RunAsync(DateTime.MinValue, 1, remainder.ToArray()));
             }
         }
     }

GetIntoTeachingApi/Models/CandidateIdsRequest.cs

@@ -0,0 +1,11 @@
+using System;
+using System.Globalization;
+using System.Linq;
+
+namespace GetIntoTeachingApi.Models
+{
+    public class CandidateIdsRequest
+    {
+        public int[] CandidateIds { get; set; }
+    }
+}

GetIntoTeachingApiTests/Controllers/OperationsControllerTests.cs

@@ -138,6 +138,43 @@ public void BackfillApplyCandidates_WhenAlreadyRunning_ReturnsBadRequest()
                 It.Is<Job>(job => job.Type == typeof(ApplyBackfillJob) && job.Method.Name == "RunAsync" && (DateTime)job.Args[0] == updatedSince),
                 It.IsAny<EnqueuedState>()),Times.Never);
         }
+
+        [Fact]
+        public void BackfillApplyCandidatesFromIds_Authorize_IsPresent()
+        {
+            typeof(OperationsController).GetMethod("BackfillApplyCandidatesFromIds")
+                .Should().BeDecoratedWith<AuthorizeAttribute>(a => a.Roles == "Admin");
+        }
+
+        [Fact]
+        public void BackfillApplyCandidatesFromIds_WhenNotAlreadyRunning_EnqueuesJob()
+        {
+            _mockAppSettings.Setup(m => m.IsApplyBackfillInProgress).Returns(false);
+            var candidateIds = new[] { 1, 2, 3 };
+            var request = new CandidateIdsRequest { CandidateIds = candidateIds };
+            var response = _controller.BackfillApplyCandidatesFromIds(request);
+
+            response.Should().BeOfType<NoContentResult>();
+
+            _mockJobClient.Verify(x => x.Create(
+                It.Is<Job>(job => job.Type == typeof(ApplyBackfillJob) && job.Method.Name == "RunAsync" && (int[])job.Args[2] == candidateIds),
+                It.IsAny<EnqueuedState>()), Times.Once);
+        }
+
+        [Fact]
+        public void BackfillApplyCandidatesFromIds_WhenAlreadyRunning_ReturnsBadRequest()
+        {
+            _mockAppSettings.Setup(m => m.IsApplyBackfillInProgress).Returns(true);
+            var candidateIds = new[] { 1, 2, 3 };
+            var request = new CandidateIdsRequest { CandidateIds = candidateIds };
+            var response = _controller.BackfillApplyCandidatesFromIds(request);
+
+            response.Should().BeOfType<BadRequestObjectResult>();
+
+            _mockJobClient.Verify(x => x.Create(
+                It.Is<Job>(job => job.Type == typeof(ApplyBackfillJob) && job.Method.Name == "RunAsync" && (int[])job.Args[2] == candidateIds),
+                It.IsAny<EnqueuedState>()),Times.Never);
+        }
 
         [Theory]
         [InlineData(true, true, true, true, true, "healthy")]

GetIntoTeachingApiTests/Jobs/ApplyBackfillJobTests.cs

@@ -86,6 +86,33 @@ public async Task RunAsync_WhenMultiplePagesAvailable_QueuesCandidateJobsForEach
                 (DateTime)job.Args[0] == updatedSince && (int)job.Args[1] == (ApplyBackfillJob.PagesPerJob + 1)),
                 It.IsAny<EnqueuedState>()), Times.Once);
         }
+
+        [Fact]
+        public async Task RunAsync_WithCandidateIds()
+        {
+            int[] candidateIds = new[] { 1,2,3 };
+            var candidate1 = new Candidate() { Id = "1", Attributes = new CandidateAttributes() { Email = $"[email protected]" } };
+            var candidate2 = new Candidate() { Id = "2", Attributes = new CandidateAttributes() { Email = $"[email protected]" } };
+            var candidate3 = new Candidate() { Id = "3", Attributes = new CandidateAttributes() { Email = $"[email protected]" } };
+
+            using (var httpTest = new HttpTest())
+            {
+                MockIndividualResponse(httpTest, new Response<Candidate>() { Data = candidate1 }, 1);
+                MockIndividualResponse(httpTest, new Response<Candidate>() { Data = candidate2 }, 2);
+                MockIndividualResponse(httpTest, new Response<Candidate>() { Data = candidate3 }, 3);
+
+                await _job.RunAsync(DateTime.MinValue, 1, candidateIds);
+            }
+
+            _mockJobClient.Verify(x => x.Create(
+                It.Is<Job>(job => job.Type == typeof(ApplyCandidateSyncJob) && job.Method.Name == "Run"),
+                It.IsAny<ScheduledState>()), Times.Exactly(3));
+
+            _mockAppSettings.VerifySet(m => m.IsApplyBackfillInProgress = true, Times.Once);
+            _mockLogger.VerifyInformationWasCalled("ApplyBackfillJob - Started - Pages 1 to 10");
+            _mockLogger.VerifyInformationWasCalled("ApplyBackfillJob - Succeeded - Pages 1 to 10");
+            _mockAppSettings.VerifySet(m => m.IsApplyBackfillInProgress = false, Times.Once);
+        }
 
         [Fact]
         public async Task RunAsync_WithNoCandidates_DoesNotQueueJobs()
@@ -122,5 +149,16 @@ private void MockResponse(HttpTest httpTest, Response<IEnumerable<Candidate>> re
                     .WithHeader("Authorization", $"Bearer {_mockEnv.Object.ApplyCandidateApiKey}")
                     .RespondWith(json, 200, headers);
         }
+
+        private void MockIndividualResponse(HttpTest httpTest, Response<Candidate> response, int candidateId)
+        {
+            var json = JsonConvert.SerializeObject(response);
+
+            httpTest
+                .ForCallsTo($"{_mockEnv.Object.ApplyCandidateApiUrl}/candidates/C{candidateId}")
+                .WithVerb("GET")
+                .WithHeader("Authorization", $"Bearer {_mockEnv.Object.ApplyCandidateApiKey}")
+                .RespondWith(json, 200);
+        }
     }
 }

GetIntoTeachingApiTests/Models/CandidateIdsRequestTests.cs

@@ -0,0 +1,17 @@
+using FluentAssertions;
+using GetIntoTeachingApi.Models;
+using Xunit;
+
+namespace GetIntoTeachingApiTests.Models
+{
+    public class CandidateIdsRequestTests
+    {
+        [Fact]
+        public void Constructor_WithInitializer()
+        {
+            var candidateIdsRequest = new CandidateIdsRequest { CandidateIds =  new[] { 1, 2, 3 }  };
+
+            candidateIdsRequest.CandidateIds.Should().Equal(1, 2, 3);
+        }
+    }
+}