diff --git a/.github/workflows/build-docker-image.yml b/.github/workflows/build-docker-image.yml index b3b3c08..0d5c4b6 100644 --- a/.github/workflows/build-docker-image.yml +++ b/.github/workflows/build-docker-image.yml @@ -1,14 +1,11 @@ name: Docker Publish - on: release: types: [published] - env: REGISTRY: ghcr.io IMAGE_NAME: ${{ github.repository }} RELEASE_TAG: ${{ github.event.release.tag_name }} - jobs: build: name: 'Docker Publish' @@ -18,18 +15,15 @@ jobs: packages: write # This is used to complete the identity challenge with sigstore/fulcio when running outside of PRs. id-token: write - steps: # Checkout the release tag version - name: Checkout repository ${{ env.RELEASE_TAG }} uses: actions/checkout@v3 with: ref: ${{ env.RELEASE_TAG }} - # Get git commit hash - name: Get short hash run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_ENV - # Need to lower case the image name for the docker tags when publishing - name: Downcase IMAGE_NAME variable run: echo "IMAGE_NAME_LOWER=${IMAGE_NAME,,}" >> $GITHUB_ENV @@ -37,19 +31,15 @@ jobs: # Sort out the image tags - name: Set initial tag run: echo "IMAGE_TAGS=${{ env.REGISTRY }}/${{ env.IMAGE_NAME_LOWER }}:${{ env.RELEASE_TAG }}" >> $GITHUB_ENV - - name: Add latest tag if we're not production release if: contains(env.RELEASE_TAG, 'next') run: echo "IMAGE_TAGS=${{ env.IMAGE_TAGS }},${{ env.REGISTRY }}/${{ env.IMAGE_NAME_LOWER }}:latest" >> $GITHUB_ENV - #debug - name: Log the tags run: echo "Calculated tags value => ${{ env.IMAGE_TAGS }}" - # Setup docker build tool - name: Setup Docker buildx uses: docker/setup-buildx-action@v3 - # Login against a Docker registry - name: Log into registry ${{ env.REGISTRY }} uses: docker/login-action@v3 @@ -57,14 +47,12 @@ jobs: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - # Extract metadata (tags, labels) for Docker - name: Extract Docker metadata id: meta uses: docker/metadata-action@v5 with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - # Build and push Docker image with Buildx (don't push on PR) - name: Build and push Docker image id: build-and-push @@ -79,8 +67,7 @@ jobs: VCSREF=${{ env.sha_short }} VCSTAG=${{ env.RELEASE_TAG }} cache-from: type=gha - cache-to: type=gha,mode=max - + cache-to: type=gha,mode=max # Sign the resulting Docker image digest. # This will only write to the public Rekor transparency log when the Docker # repository is public to avoid leaking data. If you would like to publish @@ -92,4 +79,4 @@ jobs: run: cosign version - name: Sign the published Docker image # This step uses the identity token to provision an ephemeral certificate against the sigstore community Fulcio instance. - run: echo "${{ steps.meta.outputs.tags }}" | xargs -I {} cosign sign --yes {}@${{ steps.build-and-push.outputs.digest }} + run: echo "${{ steps.meta.outputs.tags }}" | xargs -I {} cosign sign --yes {}@${{ steps.build-and-push.outputs.digest }} \ No newline at end of file diff --git a/README.md b/README.md index bdfe008..1ba1f3c 100644 --- a/README.md +++ b/README.md @@ -15,19 +15,41 @@ dotnet build dotnet test ``` -## Environment variables -The following environment variables will need to be specified for the application to run: - -| Variable name | Type/Value | Description | -| ------------- | ------------- | ------------- | -| CPD_SEARCH_API_KEY | string | The Azure AI Search API key | -| CPD_INSTRUMENTATION_CONNECTIONSTRING | string | The Azure ApplicationInsights connection string | -| VCS-TAG | string | The application version | -| CPD_SEARCH_BATCH_SIZE | integer (e.g. 10/20 etc) | The batch size for queries into Contentful | -| CPD_SEARCH_ENDPOINT | string | The Azure AI Search endpoint | -| CPD_SEARCH_INDEX_NAME | string | The Azure AI Search index name to access/create | -| CPD_DELIVERY_KEY | string | The Contentful delivery API key | -| CPD_CONTENTFUL_ENVIRONMENT | string | The Contentful enviroment id | -| CPD_SPACE_ID | string | The Contentful space id | -| CPD_SEARCH_RECREATE_INDEX_ON_REBUILD | boolean (true/false) | Whether to delete the index and recreate before populating | +## Configuration +Three configuration values are required to be set in the environment: +* ``CPD_KEY_VAULT_NAME`` - the name of the keyvault to retrieve configuration from in a deployed environment +* ``CPD_CONFIG_SECTION_NAME`` - this is the key name of the root of the configuration section that stores most of the application config +* ``__Application__Version`` - the application version, note the prefix should be set to the above value, for example if `CPD_CONFIG_SECTION_NAME` was set to `DEV` then it would be `DEV__Application__Version` +The remaining application configuration is stored within an IConfiguration section, named using (1) above. In a deployed environment this should be stored in Azure Key Vault. + +The configuration under the root key takes the following hierarchical structure: +``` + ApplicationInsights + ConnectionString + Contentful + DeliveryKey + Environment + SpaceId + SearchIndexing + ApiKey + BatchSize + Endpoint + IndexName + RecreateIndex +``` + + +### Local developer configuration +For development, this structure can be configured in the `secrets.json` file for the project. +Also make sure you set `LOCAL_ENVIRONMENT` to true, otherwise the application will try to initialise key vault. `LOCAL_ENVIRONMENT` is not required for any other scenario. + +### Deployed environments +Key names for secrets are built from the full path to the config key, separating each level with two dash characters (--). So if `CPD_CONFIG_SECTION_NAME` is set to `DEV` then it would be: +``` +DEV--ApplicationInsights--ConnectionString +DEV--ApplicationVersion +DEV--Contentful--DeliveryKey +... +``` +Note this format is for **Key Vault only**. \ No newline at end of file diff --git a/src/Childrens-Social-Care-CPD-Indexer.Tests/Core/ResourcesIndexerTest.cs b/src/Childrens-Social-Care-CPD-Indexer.Tests/Core/ResourcesIndexerTests.cs similarity index 75% rename from src/Childrens-Social-Care-CPD-Indexer.Tests/Core/ResourcesIndexerTest.cs rename to src/Childrens-Social-Care-CPD-Indexer.Tests/Core/ResourcesIndexerTests.cs index fd27f3e..b15d8df 100644 --- a/src/Childrens-Social-Care-CPD-Indexer.Tests/Core/ResourcesIndexerTest.cs +++ b/src/Childrens-Social-Care-CPD-Indexer.Tests/Core/ResourcesIndexerTests.cs @@ -1,15 +1,44 @@ using Azure; +using Azure.Core; using Azure.Search.Documents; using Azure.Search.Documents.Indexes; using Azure.Search.Documents.Indexes.Models; using Childrens_Social_Care_CPD_Indexer.Core; using Microsoft.Extensions.Logging; +using NSubstitute.ExceptionExtensions; +using System.Diagnostics.CodeAnalysis; namespace Childrens_Social_Care_CPD_Indexer.Tests.Core; -public class ResourcesIndexerTest +internal sealed class MockResponse : Response { - private ILogger _logger; + public override int Status => 404; + public override string ReasonPhrase => string.Empty; + + public override Stream? ContentStream + { + get => new MemoryStream(); + set => throw new NotImplementedException(); + } + public override string ClientRequestId + { + get => throw new NotImplementedException(); + set => throw new NotImplementedException(); + } + + public override void Dispose() => throw new NotImplementedException(); + protected override bool ContainsHeader(string name) => false; + protected override IEnumerable EnumerateHeaders() => Array.Empty(); + protected override bool TryGetHeader(string name, [NotNullWhen(true)] out string? value) { + value = null; + return false; + } + protected override bool TryGetHeaderValues(string name, [NotNullWhen(true)] out IEnumerable? values) => throw new NotImplementedException(); +} + +public class ResourcesIndexerTests +{ + private ILogger _logger; private SearchIndexClient _client; private IDocumentFetcher _documentFetcher; private ResourcesIndexer _sut; @@ -21,16 +50,15 @@ public void Setup() _client = Substitute.For(); _documentFetcher = Substitute.For(); - _sut = new ResourcesIndexer(_client, _documentFetcher, _logger); + _sut = new ResourcesIndexer(_client, _documentFetcher, _logger, MockTelemetryClient.Create()); } [Test] public async Task DeleteIndexAsync_Skips_Deletion_If_Index_Does_Not_Exist() { // arrange - var response = Substitute.For>(); - response.HasValue.Returns(false); - _client.GetIndexAsync(Arg.Any(), Arg.Any()).Returns(Task.FromResult(response)); + var exception = new RequestFailedException(new MockResponse()); + _client.GetIndexAsync(Arg.Any(), Arg.Any()).Throws(exception); // act await _sut.DeleteIndexAsync("foo"); @@ -77,9 +105,8 @@ public async Task DeleteIndexAsync_Logs_Failure_To_Delete_Index() public async Task CreateIndexAsync_Creates_The_Index() { // arrange - var getIndexResult = Substitute.For>(); - getIndexResult.HasValue.Returns(false); - _client.GetIndexAsync(Arg.Any(), Arg.Any()).Returns(Task.FromResult(getIndexResult)); + var exception = new RequestFailedException(new MockResponse()); + _client.GetIndexAsync(Arg.Any(), Arg.Any()).Throws(exception); SearchIndex? searchIndex = null; await _client.CreateIndexAsync(Arg.Do(x => searchIndex = x), Arg.Any()); @@ -141,7 +168,6 @@ public async Task PopulateIndexAsync_Uploads_Documents_In_Multiple_Batches() await _sut.PopulateIndexAsync("foo", 10); - await client.Received(2) - .UploadDocumentsAsync(documents, Arg.Any(), Arg.Any()); + await client.Received(2).UploadDocumentsAsync(documents, Arg.Any(), Arg.Any()); } } \ No newline at end of file diff --git a/src/Childrens-Social-Care-CPD-Indexer.Tests/IndexingServiceTests.cs b/src/Childrens-Social-Care-CPD-Indexer.Tests/IndexingServiceTests.cs deleted file mode 100644 index 57280f6..0000000 --- a/src/Childrens-Social-Care-CPD-Indexer.Tests/IndexingServiceTests.cs +++ /dev/null @@ -1,77 +0,0 @@ -using Childrens_Social_Care_CPD_Indexer.Core; -using Microsoft.ApplicationInsights; -using Microsoft.Extensions.Logging; -using NSubstitute.ExceptionExtensions; - -namespace Childrens_Social_Care_CPD_Indexer.Tests; - -public class IndexingServiceTests -{ - private ILogger _logger; - private IResourcesIndexerConfig _config; - private IResourcesIndexer _indexer; - private IndexingService _sut; - - [SetUp] - public void Setup() - { - _logger = Substitute.For>(); - _config = Substitute.For(); - _indexer = Substitute.For(); - _sut = new IndexingService(_indexer, _logger, _config); - } - - [Test] - public void StopAsync_Returns_Completed_Task() - { - // act - var task = _sut.StopAsync(default); - - // assert - task.IsCompleted.Should().BeTrue(); - } - - [Test] - public async Task StartAsync_Deletes_Index_If_Configured() - { - // arrange - _config.RecreateIndex.Returns(true); - - // act - await _sut.StartAsync(default); - - // assert - await _indexer.Received(1).DeleteIndexAsync(Arg.Any(), Arg.Any()); - await _indexer.Received(1).CreateIndexAsync(Arg.Any(), Arg.Any()); - } - - [Test] - public async Task StartAsync_Populates_Index() - { - // arrange - _config.RecreateIndex.Returns(false); - - // act - await _sut.StartAsync(default); - - // assert - await _indexer.Received(1).PopulateIndexAsync(Arg.Any(), Arg.Any(), Arg.Any()); - } - - [Test] - public async Task StartAsync_Logs_Exception() - { - // arrange - var exception = new InvalidOperationException(); - _config.RecreateIndex.Returns(true); - _indexer.DeleteIndexAsync(Arg.Any(), Arg.Any()).Throws(exception); - - // act - await _sut.StartAsync(default); - - // assert - _logger.Received(1).LogError(exception, "Unhandled exception occured"); - await _indexer.Received(0).CreateIndexAsync(Arg.Any(), Arg.Any()); - await _indexer.Received(0).PopulateIndexAsync(Arg.Any(), Arg.Any(), Arg.Any()); - } -} \ No newline at end of file diff --git a/src/Childrens-Social-Care-CPD-Indexer.Tests/MockTelemetryClient.cs b/src/Childrens-Social-Care-CPD-Indexer.Tests/MockTelemetryClient.cs new file mode 100644 index 0000000..1eb0a22 --- /dev/null +++ b/src/Childrens-Social-Care-CPD-Indexer.Tests/MockTelemetryClient.cs @@ -0,0 +1,32 @@ +using Microsoft.ApplicationInsights; +using Microsoft.ApplicationInsights.Channel; +using Microsoft.ApplicationInsights.Extensibility; +using System.Collections.Concurrent; + +namespace Childrens_Social_Care_CPD_Indexer.Tests; + +internal sealed class MockTelemetryChannel : ITelemetryChannel +{ + public ConcurrentBag SentTelemtries = []; + public bool IsFlushed { get; private set; } + public bool? DeveloperMode { get; set; } + public string EndpointAddress { get; set; } = string.Empty; + + public void Send(ITelemetry item) => SentTelemtries.Add(item); + public void Flush() => IsFlushed = true; + public void Dispose() {} +} + +internal static class MockTelemetryClient +{ + public static TelemetryClient Create() + { + var mockTelemetryConfig = new TelemetryConfiguration + { + TelemetryChannel = new MockTelemetryChannel(), + }; + + return new TelemetryClient(mockTelemetryConfig); + } + +} diff --git a/src/Childrens-Social-Care-CPD-Indexer.Tests/ResourcesIndexerConfigTests.cs b/src/Childrens-Social-Care-CPD-Indexer.Tests/ResourcesIndexerConfigTests.cs deleted file mode 100644 index 2790358..0000000 --- a/src/Childrens-Social-Care-CPD-Indexer.Tests/ResourcesIndexerConfigTests.cs +++ /dev/null @@ -1,58 +0,0 @@ -using Microsoft.Extensions.Configuration; - -namespace Childrens_Social_Care_CPD_Indexer.Tests; - -public class ResourcesIndexerConfigTests -{ - [TestCase("ApiKey", "CPD_SEARCH_API_KEY", "Api key", "Api key")] - [TestCase("AppInsightsConnectionString", "CPD_INSTRUMENTATION_CONNECTIONSTRING", "Connection string", "Connection string")] - [TestCase("ApplicationVersion", "VCS-TAG", "1.0.0", "1.0.0")] - [TestCase("BatchSize", "CPD_SEARCH_BATCH_SIZE", "2", 2)] - [TestCase("Endpoint", "CPD_SEARCH_ENDPOINT", "Endpoint", "Endpoint")] - [TestCase("IndexName", "CPD_SEARCH_INDEX_NAME", "Index name", "Index name")] - [TestCase("ContentfulApiKey", "CPD_DELIVERY_KEY", "Contentful api key", "Contentful api key")] - [TestCase("ContentfulEnvironmentId", "CPD_CONTENTFUL_ENVIRONMENT", "Environment id", "Environment id")] - [TestCase("ContentfulSpaceId", "CPD_SPACE_ID", "Space id", "Space id")] - [TestCase("RecreateIndex", "CPD_SEARCH_RECREATE_INDEX_ON_REBUILD", "true", true)] - [TestCase("RecreateIndex", "CPD_SEARCH_RECREATE_INDEX_ON_REBUILD", "false", false)] - public void Config_Returns_Values(string propName, string key, string value, object expected) - { - // arrange - var inMemorySettings = new Dictionary { - {key, value}, - }; - var configuration = new ConfigurationBuilder().AddInMemoryCollection(inMemorySettings!).Build(); - var sut = new ResourcesIndexerConfig(configuration); - var propertyInfo = typeof(ResourcesIndexerConfig).Properties().Single(x => x.Name == propName); - - // act - var actual = propertyInfo.GetValue(sut); - - // assert - actual.Should().Be(expected); - } - - [TestCase("ApiKey", "")] - [TestCase("AppInsightsConnectionString", "")] - [TestCase("ApplicationVersion", "")] - [TestCase("BatchSize", 20)] - [TestCase("Endpoint", "")] - [TestCase("IndexName", "")] - [TestCase("ContentfulApiKey", "")] - [TestCase("ContentfulEnvironmentId", "")] - [TestCase("ContentfulSpaceId", "")] - [TestCase("RecreateIndex", true)] - public void Config_Returns_Default_Values(string propName, object expected) - { - // arrange - var configuration = new ConfigurationBuilder().AddInMemoryCollection(new Dictionary()!).Build(); - var sut = new ResourcesIndexerConfig(configuration); - var propertyInfo = typeof(ResourcesIndexerConfig).Properties().Single(x => x.Name == propName); - - // act - var actual = propertyInfo.GetValue(sut); - - // assert - actual.Should().Be(expected); - } -} \ No newline at end of file diff --git a/src/Childrens-Social-Care-CPD-Indexer.Tests/WorkerTests.cs b/src/Childrens-Social-Care-CPD-Indexer.Tests/WorkerTests.cs new file mode 100644 index 0000000..1bbab38 --- /dev/null +++ b/src/Childrens-Social-Care-CPD-Indexer.Tests/WorkerTests.cs @@ -0,0 +1,91 @@ +using Childrens_Social_Care_CPD_Indexer.Core; +using Microsoft.ApplicationInsights.Channel; +using Microsoft.ApplicationInsights.Extensibility; +using Microsoft.ApplicationInsights; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using NSubstitute.ExceptionExtensions; + +namespace Childrens_Social_Care_CPD_Indexer.Tests; + +public class WorkerTests +{ + private ILogger _logger; + private IApplicationConfiguration _config; + private IResourcesIndexer _indexer; + private IHostApplicationLifetime _hostingApplicationLifetime; + private TelemetryClient _telemetryClient; + private Worker _sut; + + [SetUp] + public void Setup() + { + _logger = Substitute.For>(); + _config = Substitute.For(); + _indexer = Substitute.For(); + _hostingApplicationLifetime = Substitute.For(); + + var configuration = new TelemetryConfiguration(); + var sendItems = new List(); + var channel = Substitute.For(); + configuration.TelemetryChannel = channel; + configuration.ConnectionString = "InstrumentationKey={Guid.NewGuid()};"; + configuration.TelemetryInitializers.Add(new OperationCorrelationTelemetryInitializer()); + _telemetryClient = new TelemetryClient(configuration); + + _sut = new Worker(_logger, _indexer, _config, _hostingApplicationLifetime, _telemetryClient); + } + + [TearDown] + public void Teardown() + { + _sut.Dispose(); + } + + [Test] + public async Task StartAsync_Deletes_Index_If_Configured() + { + // arrange + _config.SearchIndexing.RecreateIndex.Returns(true); + var cancellationTokenSource = new CancellationTokenSource(); + + // act + await _sut.StartAsync(cancellationTokenSource.Token); + + // assert + await _indexer.Received(1).DeleteIndexAsync(Arg.Any(), Arg.Any()); + await _indexer.Received(1).CreateIndexAsync(Arg.Any(), Arg.Any()); + } + + [Test] + public async Task StartAsync_Populates_Index() + { + // arrange + _config.SearchIndexing.RecreateIndex.Returns(false); + var cancellationTokenSource = new CancellationTokenSource(); + + // act + await _sut.StartAsync(cancellationTokenSource.Token); + + // assert + await _indexer.Received(1).PopulateIndexAsync(Arg.Any(), Arg.Any(), Arg.Any()); + } + + [Test] + public async Task StartAsync_Logs_Exception() + { + // arrange + var exception = new InvalidOperationException(); + _config.SearchIndexing.RecreateIndex.Returns(true); + _indexer.DeleteIndexAsync(Arg.Any(), Arg.Any()).Throws(exception); + var cancellationTokenSource = new CancellationTokenSource(); + + // act + await _sut.StartAsync(cancellationTokenSource.Token); + + // assert + _logger.Received(1).LogError(exception, "Unhandled exception occured"); + await _indexer.Received(0).CreateIndexAsync(Arg.Any(), Arg.Any()); + await _indexer.Received(0).PopulateIndexAsync(Arg.Any(), Arg.Any(), Arg.Any()); + } +} \ No newline at end of file diff --git a/src/Childrens-Social-Care-CPD-Indexer.sln b/src/Childrens-Social-Care-CPD-Indexer.sln index ede56d3..0a76fb1 100644 --- a/src/Childrens-Social-Care-CPD-Indexer.sln +++ b/src/Childrens-Social-Care-CPD-Indexer.sln @@ -1,11 +1,11 @@  Microsoft Visual Studio Solution File, Format Version 12.00 # Visual Studio Version 17 -VisualStudioVersion = 17.8.34322.80 +VisualStudioVersion = 17.8.34330.188 MinimumVisualStudioVersion = 10.0.40219.1 -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Childrens-Social-Care-CPD-Indexer", "Childrens-Social-Care-CPD-Indexer\Childrens-Social-Care-CPD-Indexer.csproj", "{D806CCEA-5505-4CD9-98B6-08A5F161C06A}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Childrens-Social-Care-CPD-Indexer", "Childrens-Social-Care-CPD-Indexer\Childrens-Social-Care-CPD-Indexer.csproj", "{F2710DEC-7B59-484A-9D04-F73B1E09563A}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Childrens-Social-Care-CPD-Indexer.Tests", "Childrens-Social-Care-CPD-Indexer.Tests\Childrens-Social-Care-CPD-Indexer.Tests.csproj", "{6984BF56-808E-4294-949D-FFE4B02CCE16}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Childrens-Social-Care-CPD-Indexer.Tests", "Childrens-Social-Care-CPD-Indexer.Tests\Childrens-Social-Care-CPD-Indexer.Tests.csproj", "{F0344BA1-9B92-42A7-B650-B8064EDD2B9E}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -13,19 +13,19 @@ Global Release|Any CPU = Release|Any CPU EndGlobalSection GlobalSection(ProjectConfigurationPlatforms) = postSolution - {D806CCEA-5505-4CD9-98B6-08A5F161C06A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {D806CCEA-5505-4CD9-98B6-08A5F161C06A}.Debug|Any CPU.Build.0 = Debug|Any CPU - {D806CCEA-5505-4CD9-98B6-08A5F161C06A}.Release|Any CPU.ActiveCfg = Release|Any CPU - {D806CCEA-5505-4CD9-98B6-08A5F161C06A}.Release|Any CPU.Build.0 = Release|Any CPU - {6984BF56-808E-4294-949D-FFE4B02CCE16}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {6984BF56-808E-4294-949D-FFE4B02CCE16}.Debug|Any CPU.Build.0 = Debug|Any CPU - {6984BF56-808E-4294-949D-FFE4B02CCE16}.Release|Any CPU.ActiveCfg = Release|Any CPU - {6984BF56-808E-4294-949D-FFE4B02CCE16}.Release|Any CPU.Build.0 = Release|Any CPU + {F2710DEC-7B59-484A-9D04-F73B1E09563A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {F2710DEC-7B59-484A-9D04-F73B1E09563A}.Debug|Any CPU.Build.0 = Debug|Any CPU + {F2710DEC-7B59-484A-9D04-F73B1E09563A}.Release|Any CPU.ActiveCfg = Release|Any CPU + {F2710DEC-7B59-484A-9D04-F73B1E09563A}.Release|Any CPU.Build.0 = Release|Any CPU + {F0344BA1-9B92-42A7-B650-B8064EDD2B9E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {F0344BA1-9B92-42A7-B650-B8064EDD2B9E}.Debug|Any CPU.Build.0 = Debug|Any CPU + {F0344BA1-9B92-42A7-B650-B8064EDD2B9E}.Release|Any CPU.ActiveCfg = Release|Any CPU + {F0344BA1-9B92-42A7-B650-B8064EDD2B9E}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution - SolutionGuid = {EEC0CA75-7434-4AC0-9F45-31CA6CC7D59A} + SolutionGuid = {5F3E3B0D-C6EA-4304-851B-75680DAB332E} EndGlobalSection EndGlobal diff --git a/src/Childrens-Social-Care-CPD-Indexer/.gitignore b/src/Childrens-Social-Care-CPD-Indexer/.gitignore new file mode 100644 index 0000000..ff5b00c --- /dev/null +++ b/src/Childrens-Social-Care-CPD-Indexer/.gitignore @@ -0,0 +1,264 @@ +## Ignore Visual Studio temporary files, build results, and +## files generated by popular Visual Studio add-ons. + +# Azure Functions localsettings file +local.settings.json + +# User-specific files +*.suo +*.user +*.userosscache +*.sln.docstates + +# User-specific files (MonoDevelop/Xamarin Studio) +*.userprefs + +# Build results +[Dd]ebug/ +[Dd]ebugPublic/ +[Rr]elease/ +[Rr]eleases/ +x64/ +x86/ +bld/ +[Bb]in/ +[Oo]bj/ +[Ll]og/ + +# Visual Studio 2015 cache/options directory +.vs/ +# Uncomment if you have tasks that create the project's static files in wwwroot +#wwwroot/ + +# MSTest test Results +[Tt]est[Rr]esult*/ +[Bb]uild[Ll]og.* + +# NUNIT +*.VisualState.xml +TestResult.xml + +# Build Results of an ATL Project +[Dd]ebugPS/ +[Rr]eleasePS/ +dlldata.c + +# DNX +project.lock.json +project.fragment.lock.json +artifacts/ + +*_i.c +*_p.c +*_i.h +*.ilk +*.meta +*.obj +*.pch +*.pdb +*.pgc +*.pgd +*.rsp +*.sbr +*.tlb +*.tli +*.tlh +*.tmp +*.tmp_proj +*.log +*.vspscc +*.vssscc +.builds +*.pidb +*.svclog +*.scc + +# Chutzpah Test files +_Chutzpah* + +# Visual C++ cache files +ipch/ +*.aps +*.ncb +*.opendb +*.opensdf +*.sdf +*.cachefile +*.VC.db +*.VC.VC.opendb + +# Visual Studio profiler +*.psess +*.vsp +*.vspx +*.sap + +# TFS 2012 Local Workspace +$tf/ + +# Guidance Automation Toolkit +*.gpState + +# ReSharper is a .NET coding add-in +_ReSharper*/ +*.[Rr]e[Ss]harper +*.DotSettings.user + +# JustCode is a .NET coding add-in +.JustCode + +# TeamCity is a build add-in +_TeamCity* + +# DotCover is a Code Coverage Tool +*.dotCover + +# NCrunch +_NCrunch_* +.*crunch*.local.xml +nCrunchTemp_* + +# MightyMoose +*.mm.* +AutoTest.Net/ + +# Web workbench (sass) +.sass-cache/ + +# Installshield output folder +[Ee]xpress/ + +# DocProject is a documentation generator add-in +DocProject/buildhelp/ +DocProject/Help/*.HxT +DocProject/Help/*.HxC +DocProject/Help/*.hhc +DocProject/Help/*.hhk +DocProject/Help/*.hhp +DocProject/Help/Html2 +DocProject/Help/html + +# Click-Once directory +publish/ + +# Publish Web Output +*.[Pp]ublish.xml +*.azurePubxml +# TODO: Comment the next line if you want to checkin your web deploy settings +# but database connection strings (with potential passwords) will be unencrypted +#*.pubxml +*.publishproj + +# Microsoft Azure Web App publish settings. Comment the next line if you want to +# checkin your Azure Web App publish settings, but sensitive information contained +# in these scripts will be unencrypted +PublishScripts/ + +# NuGet Packages +*.nupkg +# The packages folder can be ignored because of Package Restore +**/packages/* +# except build/, which is used as an MSBuild target. +!**/packages/build/ +# Uncomment if necessary however generally it will be regenerated when needed +#!**/packages/repositories.config +# NuGet v3's project.json files produces more ignoreable files +*.nuget.props +*.nuget.targets + +# Microsoft Azure Build Output +csx/ +*.build.csdef + +# Microsoft Azure Emulator +ecf/ +rcf/ + +# Windows Store app package directories and files +AppPackages/ +BundleArtifacts/ +Package.StoreAssociation.xml +_pkginfo.txt + +# Visual Studio cache files +# files ending in .cache can be ignored +*.[Cc]ache +# but keep track of directories ending in .cache +!*.[Cc]ache/ + +# Others +ClientBin/ +~$* +*~ +*.dbmdl +*.dbproj.schemaview +*.jfm +*.pfx +*.publishsettings +node_modules/ +orleans.codegen.cs + +# Since there are multiple workflows, uncomment next line to ignore bower_components +# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) +#bower_components/ + +# RIA/Silverlight projects +Generated_Code/ + +# Backup & report files from converting an old project file +# to a newer Visual Studio version. Backup files are not needed, +# because we have git ;-) +_UpgradeReport_Files/ +Backup*/ +UpgradeLog*.XML +UpgradeLog*.htm + +# SQL Server files +*.mdf +*.ldf + +# Business Intelligence projects +*.rdl.data +*.bim.layout +*.bim_*.settings + +# Microsoft Fakes +FakesAssemblies/ + +# GhostDoc plugin setting file +*.GhostDoc.xml + +# Node.js Tools for Visual Studio +.ntvs_analysis.dat + +# Visual Studio 6 build log +*.plg + +# Visual Studio 6 workspace options file +*.opt + +# Visual Studio LightSwitch build output +**/*.HTMLClient/GeneratedArtifacts +**/*.DesktopClient/GeneratedArtifacts +**/*.DesktopClient/ModelManifest.xml +**/*.Server/GeneratedArtifacts +**/*.Server/ModelManifest.xml +_Pvt_Extensions + +# Paket dependency manager +.paket/paket.exe +paket-files/ + +# FAKE - F# Make +.fake/ + +# JetBrains Rider +.idea/ +*.sln.iml + +# CodeRush +.cr/ + +# Python Tools for Visual Studio (PTVS) +__pycache__/ +*.pyc \ No newline at end of file diff --git a/src/Childrens-Social-Care-CPD-Indexer/ApplicationConfiguration.cs b/src/Childrens-Social-Care-CPD-Indexer/ApplicationConfiguration.cs new file mode 100644 index 0000000..11aa35a --- /dev/null +++ b/src/Childrens-Social-Care-CPD-Indexer/ApplicationConfiguration.cs @@ -0,0 +1,65 @@ +using System.Diagnostics.CodeAnalysis; + +namespace Childrens_Social_Care_CPD_Indexer; + +public interface IApplicationInsightsConfig +{ + string ConnectionString { get; set; } +} + +public interface IContentfulConfig +{ + string DeliveryKey { get; set; } + string Environment { get; set; } + string SpaceId { get; set; } +} + +public interface ISearchIndexingConfig +{ + string ApiKey { get; set; } + int BatchSize { get; set; } + string Endpoint { get; set; } + string IndexName { get; set; } + bool RecreateIndex { get; set; } +} + +public interface IApplicationConfiguration +{ + string ApplicationVersion { get; set; } + IApplicationInsightsConfig ApplicationInsights { get; set; } + IContentfulConfig Contentful { get; set; } + ISearchIndexingConfig SearchIndexing { get; set; } +} + +[ExcludeFromCodeCoverage] +internal class ApplicationInsightsConfig : IApplicationInsightsConfig +{ + public string ConnectionString { get; set; } = string.Empty; +} + +[ExcludeFromCodeCoverage] +internal class ContentfulConfig: IContentfulConfig +{ + public string DeliveryKey { get; set; } = string.Empty; + public string Environment { get; set; } = string.Empty; + public string SpaceId { get; set; } = string.Empty; +} + +[ExcludeFromCodeCoverage] +internal class SearchIndexingConfig: ISearchIndexingConfig +{ + public string ApiKey { get; set; } = string.Empty; + public int BatchSize { get; set; } = 25; + public string Endpoint { get; set; } = string.Empty; + public string IndexName { get; set; } = string.Empty; + public bool RecreateIndex { get; set; } = false; +} + +[ExcludeFromCodeCoverage] +internal class ApplicationConfiguration: IApplicationConfiguration +{ + public IApplicationInsightsConfig ApplicationInsights { get; set; } = new ApplicationInsightsConfig(); + public string ApplicationVersion { get; set; } = string.Empty; + public IContentfulConfig Contentful { get; set; } = new ContentfulConfig(); + public ISearchIndexingConfig SearchIndexing { get; set; } = new SearchIndexingConfig(); +} \ No newline at end of file diff --git a/src/Childrens-Social-Care-CPD-Indexer/Childrens-Social-Care-CPD-Indexer.csproj b/src/Childrens-Social-Care-CPD-Indexer/Childrens-Social-Care-CPD-Indexer.csproj index 48ae2fb..7ce2916 100644 --- a/src/Childrens-Social-Care-CPD-Indexer/Childrens-Social-Care-CPD-Indexer.csproj +++ b/src/Childrens-Social-Care-CPD-Indexer/Childrens-Social-Care-CPD-Indexer.csproj @@ -4,23 +4,23 @@ net8.0 enable enable - dotnet-Childrens_Social_Care_CPD_Indexer-5181f816-6a9f-4c7a-a524-576a7d37d013 + dotnet-Childrens_Social_Care_CPD_Indexer-907bada8-7f54-479d-b04e-45ec4f148332 Childrens_Social_Care_CPD_Indexer Linux - - - - - + + + + + + - - + - - + + diff --git a/src/Childrens-Social-Care-CPD-Indexer/Core/Content.cs b/src/Childrens-Social-Care-CPD-Indexer/Core/Content.cs index 3368689..81a6bc1 100644 --- a/src/Childrens-Social-Care-CPD-Indexer/Core/Content.cs +++ b/src/Childrens-Social-Care-CPD-Indexer/Core/Content.cs @@ -25,6 +25,7 @@ internal class Content : IContent public List? Items { get; set; } public IContent? Navigation { get; set; } public IContent? RelatedContent { get; set; } + public int? EstimatedReadingTime { get; set; } [JsonProperty("$metadata")] public ContentfulMetadata? Metadata { get; set; } diff --git a/src/Childrens-Social-Care-CPD-Indexer/Core/CpdDocument.cs b/src/Childrens-Social-Care-CPD-Indexer/Core/CpdDocument.cs index b1df9da..b43c1ae 100644 --- a/src/Childrens-Social-Care-CPD-Indexer/Core/CpdDocument.cs +++ b/src/Childrens-Social-Care-CPD-Indexer/Core/CpdDocument.cs @@ -11,7 +11,7 @@ internal partial class CpdDocument(string id) [SimpleField(IsKey = true)] public string? Id { get; set; } = Base64UrlEncoder.Encode(id); - [SimpleField()] + [SearchableField(AnalyzerName = LexicalAnalyzerName.Values.EnLucene)] public string? Title { get; set; } [SimpleField(IsFilterable = true, IsFacetable = true)] @@ -20,12 +20,15 @@ internal partial class CpdDocument(string id) [SearchableField(AnalyzerName = LexicalAnalyzerName.Values.EnLucene)] public string? Body { get; set; } - [SimpleField(IsSortable = true, IsFilterable = true)] + [SimpleField(IsSortable = true)] public DateTimeOffset? CreatedAt { get; set; } - [SimpleField(IsSortable = true, IsFilterable = true)] + [SimpleField(IsSortable = true)] public DateTimeOffset? UpdatedAt { get; set; } - + + [SimpleField] + public int? EstimatedReadingTime { get; set; } + [SimpleField(IsFilterable = true, IsFacetable = true)] public IEnumerable? Tags { get; set; } } \ No newline at end of file diff --git a/src/Childrens-Social-Care-CPD-Indexer/Core/DocumentFetcher.cs b/src/Childrens-Social-Care-CPD-Indexer/Core/DocumentFetcher.cs index 95ccd91..9a961f7 100644 --- a/src/Childrens-Social-Care-CPD-Indexer/Core/DocumentFetcher.cs +++ b/src/Childrens-Social-Care-CPD-Indexer/Core/DocumentFetcher.cs @@ -30,6 +30,7 @@ private static CpdDocument BuildResourceDocument(Content content) Body = content.SearchSummary, CreatedAt = content.Sys!.CreatedAt.HasValue ? new DateTimeOffset(content.Sys.CreatedAt.Value) : null, UpdatedAt = content.Sys!.UpdatedAt.HasValue ? new DateTimeOffset(content.Sys.UpdatedAt.Value) : null, + EstimatedReadingTime = content.EstimatedReadingTime, Tags = tags }; } diff --git a/src/Childrens-Social-Care-CPD-Indexer/Core/IResourcesIndexer.cs b/src/Childrens-Social-Care-CPD-Indexer/Core/IResourcesIndexer.cs index 13c2707..f519948 100644 --- a/src/Childrens-Social-Care-CPD-Indexer/Core/IResourcesIndexer.cs +++ b/src/Childrens-Social-Care-CPD-Indexer/Core/IResourcesIndexer.cs @@ -1,6 +1,6 @@ namespace Childrens_Social_Care_CPD_Indexer.Core; -internal interface IResourcesIndexer +public interface IResourcesIndexer { Task CreateIndexAsync(string indexName, CancellationToken cancellationToken); Task DeleteIndexAsync(string indexName, CancellationToken cancellationToken); diff --git a/src/Childrens-Social-Care-CPD-Indexer/Core/IResourcesIndexerConfig.cs b/src/Childrens-Social-Care-CPD-Indexer/Core/IResourcesIndexerConfig.cs deleted file mode 100644 index 9f59606..0000000 --- a/src/Childrens-Social-Care-CPD-Indexer/Core/IResourcesIndexerConfig.cs +++ /dev/null @@ -1,15 +0,0 @@ -namespace Childrens_Social_Care_CPD_Indexer.Core; - -internal interface IResourcesIndexerConfig -{ - string ApiKey { get; } - string AppInsightsConnectionString { get; } - string ApplicationVersion { get; } - int BatchSize { get; } - string Endpoint { get; } - string IndexName { get; } - string ContentfulApiKey { get; } - string ContentfulEnvironmentId { get; } - string ContentfulSpaceId { get; } - bool RecreateIndex { get; } -} \ No newline at end of file diff --git a/src/Childrens-Social-Care-CPD-Indexer/Core/ResourcesIndexer.cs b/src/Childrens-Social-Care-CPD-Indexer/Core/ResourcesIndexer.cs index 1c3124d..408acf7 100644 --- a/src/Childrens-Social-Care-CPD-Indexer/Core/ResourcesIndexer.cs +++ b/src/Childrens-Social-Care-CPD-Indexer/Core/ResourcesIndexer.cs @@ -1,14 +1,38 @@ -using Azure.Search.Documents.Indexes; +using Azure; +using Azure.Search.Documents.Indexes; using Azure.Search.Documents.Indexes.Models; +using Microsoft.ApplicationInsights; +using Microsoft.ApplicationInsights.DataContracts; +using Microsoft.Extensions.Logging; namespace Childrens_Social_Care_CPD_Indexer.Core; -internal class ResourcesIndexer(SearchIndexClient searchIndexClient, IDocumentFetcher documentFetcher, ILogger logger): IResourcesIndexer +internal class ResourcesIndexer(SearchIndexClient searchIndexClient, IDocumentFetcher documentFetcher, ILogger logger, TelemetryClient telemetryClient) : IResourcesIndexer { + private async Task IndexExistsAsync(string indexName, CancellationToken cancellationToken) + { + using var operation = telemetryClient.StartOperation("GetIndexAsync"); + try + { + var index = await searchIndexClient.GetIndexAsync(indexName, cancellationToken); + return index.HasValue; + } + catch (RequestFailedException rf) + { + if (rf.Status == 404) + { + return false; + } + telemetryClient.TrackException(rf); + throw; + } + } + public async Task CreateIndexAsync(string indexName, CancellationToken cancellationToken = default) { - var index = await searchIndexClient.GetIndexAsync(indexName, cancellationToken); - if (index.HasValue) + using var operation = telemetryClient.StartOperation("CreateIndexAsync"); + var indexExists = await IndexExistsAsync(indexName, cancellationToken); + if (indexExists) { logger.LogInformation("Index already exists, skipping creation."); return; @@ -24,9 +48,10 @@ public async Task CreateIndexAsync(string indexName, CancellationToken cancellat public async Task DeleteIndexAsync(string indexName, CancellationToken cancellationToken = default) { + using var operation = telemetryClient.StartOperation("DeleteIndexAsync"); logger.LogInformation("Deleting index..."); - var index = await searchIndexClient.GetIndexAsync(indexName, cancellationToken); - if (index.HasValue) + var indexExists = await IndexExistsAsync(indexName, cancellationToken); + if (indexExists) { var deleteResponse = await searchIndexClient.DeleteIndexAsync(indexName, cancellationToken); if (deleteResponse.IsError) @@ -39,9 +64,9 @@ public async Task DeleteIndexAsync(string indexName, CancellationToken cancellat public async Task PopulateIndexAsync(string indexName, int batchSize, CancellationToken cancellationToken = default) { + using var operation = telemetryClient.StartOperation("PopulateIndexAsync"); logger.LogInformation("Populating index..."); var searchClient = searchIndexClient.GetSearchClient(indexName); - var skip = 0; var batch = await documentFetcher.FetchBatchAsync(batchSize, skip, cancellationToken); @@ -51,6 +76,7 @@ public async Task PopulateIndexAsync(string indexName, int batchSize, Cancellati await searchClient.UploadDocumentsAsync(batch, null, cancellationToken); cancellationToken.ThrowIfCancellationRequested(); logger.LogInformation("Uploaded batch to index"); + telemetryClient.TrackEvent("Uploaded batch"); skip += batch.Length; batch = await documentFetcher.FetchBatchAsync(batchSize, skip, cancellationToken); } diff --git a/src/Childrens-Social-Care-CPD-Indexer/IndexingService.cs b/src/Childrens-Social-Care-CPD-Indexer/IndexingService.cs deleted file mode 100644 index b6b3a8c..0000000 --- a/src/Childrens-Social-Care-CPD-Indexer/IndexingService.cs +++ /dev/null @@ -1,37 +0,0 @@ -using Microsoft.ApplicationInsights.DataContracts; -using Microsoft.ApplicationInsights; -using Childrens_Social_Care_CPD_Indexer.Core; - -namespace Childrens_Social_Care_CPD_Indexer; - -internal class IndexingService(IResourcesIndexer resourcesIndexer, ILogger logger, IResourcesIndexerConfig config) : IHostedService -{ - public async Task StartAsync(CancellationToken cancellationToken) - { - logger.LogInformation("Indexing started at: {startTime}", DateTime.Now); - try - { - if (config.RecreateIndex) - { - await resourcesIndexer.DeleteIndexAsync(config.IndexName, cancellationToken); - } - - await resourcesIndexer.CreateIndexAsync(config.IndexName, cancellationToken); - await resourcesIndexer.PopulateIndexAsync(config.IndexName, config.BatchSize, cancellationToken); - - } - catch (Exception ex) - { - logger.LogError(ex, "Unhandled exception occured"); - } - finally - { - logger.LogInformation("Indexing finished at: {finishTime}", DateTime.Now); - } - } - - public Task StopAsync(CancellationToken cancellationToken) - { - return Task.CompletedTask; - } -} diff --git a/src/Childrens-Social-Care-CPD-Indexer/Program.cs b/src/Childrens-Social-Care-CPD-Indexer/Program.cs index 5dc3f15..ce85d1f 100644 --- a/src/Childrens-Social-Care-CPD-Indexer/Program.cs +++ b/src/Childrens-Social-Care-CPD-Indexer/Program.cs @@ -1,58 +1,66 @@ -using System.Diagnostics.CodeAnalysis; -using Microsoft.ApplicationInsights.WorkerService; -using Childrens_Social_Care_CPD_Indexer.Core; -using Childrens_Social_Care_CPD_Indexer; -using Contentful.Core; -using Microsoft.Extensions.DependencyInjection.Extensions; -using Contentful.Core.Configuration; using Azure.Search.Documents.Indexes; using Azure; +using Childrens_Social_Care_CPD_Indexer; +using Childrens_Social_Care_CPD_Indexer.Core; +using Contentful.Core.Configuration; +using Contentful.Core; +using Microsoft.ApplicationInsights.WorkerService; +using Microsoft.ApplicationInsights; +using System.Diagnostics.CodeAnalysis; +using Azure.Identity; + +var builder = Host.CreateApplicationBuilder(args); + +// Configuration +var applicationConfiguration = new ApplicationConfiguration(); + +if (!builder.Configuration.GetValue("LOCAL_ENVIRONMENT")) +{ + var keyVaultUri = new Uri($"https://{builder.Configuration.GetValue("CPD_KEY_VAULT_NAME") ?? string.Empty}.vault.azure.net/"); + builder.Configuration.AddAzureKeyVault(keyVaultUri, new DefaultAzureCredential()); +} -var builder = Host.CreateDefaultBuilder(args); +builder.Configuration.Bind("IndexingConfig", applicationConfiguration); +builder.Services.AddSingleton(applicationConfiguration); +// Logging -builder.ConfigureServices((context, services) => +var options = new ApplicationInsightsServiceOptions() { - var config = new ResourcesIndexerConfig(context.Configuration); - services.AddTransient(); + ApplicationVersion = applicationConfiguration.ApplicationVersion, + ConnectionString = applicationConfiguration.ApplicationInsights.ConnectionString, +}; - var options = new ApplicationInsightsServiceOptions +builder.Services.AddApplicationInsightsTelemetryWorkerService(options); + +// Code dependencies +builder.Services.AddTransient(); +builder.Services.AddTransient(servicesProvider => { + var httpClient = servicesProvider.GetRequiredService(); + var applicationConfiguration = servicesProvider.GetRequiredService(); + var contentfulOptions = new ContentfulOptions() { - ApplicationVersion = config.ApplicationVersion, - ConnectionString = config.AppInsightsConnectionString + DeliveryApiKey = applicationConfiguration.Contentful.DeliveryKey, + SpaceId = applicationConfiguration.Contentful.SpaceId, + Environment = applicationConfiguration.Contentful.Environment }; + return new ContentfulClient(httpClient, contentfulOptions); +}); - services.AddApplicationInsightsTelemetryWorkerService(options); - services.TryAddTransient(); - services.AddTransient(servicesProvider => { - var httpClient = servicesProvider.GetRequiredService(); - var resourcesIndexerConfig = servicesProvider.GetRequiredService(); - var contentfulOptions = new ContentfulOptions() - { - DeliveryApiKey = resourcesIndexerConfig.ContentfulApiKey, - SpaceId = resourcesIndexerConfig.ContentfulSpaceId, - Environment = resourcesIndexerConfig.ContentfulEnvironmentId - }; - return new ContentfulClient(httpClient, contentfulOptions); - }); - services.AddTransient(); - services.AddTransient(servicesProvider => { - var logger = servicesProvider.GetRequiredService>(); - var config = servicesProvider.GetRequiredService(); - var documentFetcher = servicesProvider.GetRequiredService(); - var searchEndpointUri = new Uri(config.Endpoint); - var searchIndexClient = new SearchIndexClient(searchEndpointUri, new AzureKeyCredential(config.ApiKey)); - return new ResourcesIndexer(searchIndexClient, documentFetcher, logger); - }); - - services.AddHostedService(); +builder.Services.AddTransient(); +builder.Services.AddTransient(servicesProvider => { + var logger = servicesProvider.GetRequiredService>(); + var applicationConfiguration = servicesProvider.GetRequiredService(); + var documentFetcher = servicesProvider.GetRequiredService(); + var searchEndpointUri = new Uri(applicationConfiguration.SearchIndexing.Endpoint); + var searchIndexClient = new SearchIndexClient(searchEndpointUri, new AzureKeyCredential(applicationConfiguration.SearchIndexing.ApiKey)); + var telemtryClient = servicesProvider.GetRequiredService(); + return new ResourcesIndexer(searchIndexClient, documentFetcher, logger, telemtryClient); }); -using (var host = builder.Build()) -{ - var lifetime = host.Services.GetRequiredService(); - await host.StartAsync().ContinueWith(x => lifetime.StopApplication()); -} +builder.Services.AddHostedService(); + +var host = builder.Build(); +await host.RunAsync(); [ExcludeFromCodeCoverage] -public partial class Program() -{} \ No newline at end of file +public partial class Program() { } \ No newline at end of file diff --git a/src/Childrens-Social-Care-CPD-Indexer/ResourcesIndexerConfig.cs b/src/Childrens-Social-Care-CPD-Indexer/ResourcesIndexerConfig.cs deleted file mode 100644 index d67a842..0000000 --- a/src/Childrens-Social-Care-CPD-Indexer/ResourcesIndexerConfig.cs +++ /dev/null @@ -1,17 +0,0 @@ -using Childrens_Social_Care_CPD_Indexer.Core; - -namespace Childrens_Social_Care_CPD_Indexer; - -internal class ResourcesIndexerConfig(IConfiguration configuration) : IResourcesIndexerConfig -{ - public string ApiKey => configuration.GetValue("CPD_SEARCH_API_KEY", string.Empty)!; - public string AppInsightsConnectionString => configuration.GetValue("CPD_INSTRUMENTATION_CONNECTIONSTRING", string.Empty)!; - public string ApplicationVersion => configuration.GetValue("VCS-TAG", string.Empty)!; - public int BatchSize => configuration.GetValue("CPD_SEARCH_BATCH_SIZE", 20); - public string Endpoint => configuration.GetValue("CPD_SEARCH_ENDPOINT", string.Empty)!; - public string IndexName => configuration.GetValue("CPD_SEARCH_INDEX_NAME", string.Empty)!; - public string ContentfulApiKey => configuration.GetValue("CPD_DELIVERY_KEY", string.Empty)!; - public string ContentfulEnvironmentId => configuration.GetValue("CPD_CONTENTFUL_ENVIRONMENT", string.Empty)!; - public string ContentfulSpaceId => configuration.GetValue("CPD_SPACE_ID", string.Empty)!; - public bool RecreateIndex => configuration.GetValue("CPD_SEARCH_RECREATE_INDEX_ON_REBUILD", true); -} diff --git a/src/Childrens-Social-Care-CPD-Indexer/Worker.cs b/src/Childrens-Social-Care-CPD-Indexer/Worker.cs new file mode 100644 index 0000000..ae19c8a --- /dev/null +++ b/src/Childrens-Social-Care-CPD-Indexer/Worker.cs @@ -0,0 +1,33 @@ +using Childrens_Social_Care_CPD_Indexer.Core; +using Microsoft.ApplicationInsights; + +namespace Childrens_Social_Care_CPD_Indexer; + +public class Worker(ILogger logger, IResourcesIndexer resourcesIndexer, IApplicationConfiguration applicationConfiguration, IHostApplicationLifetime hostApplicationLifetime, TelemetryClient telemetryClient) : BackgroundService +{ + protected override async Task ExecuteAsync(CancellationToken stoppingToken) => await DoWork(stoppingToken).ContinueWith(task => hostApplicationLifetime.StopApplication(), stoppingToken); + + private async Task DoWork(CancellationToken stoppingToken) + { + logger.LogInformation("Indexing started at: {startTime}", DateTime.Now); + try + { + if (applicationConfiguration.SearchIndexing.RecreateIndex) + { + await resourcesIndexer.DeleteIndexAsync(applicationConfiguration.SearchIndexing.IndexName, stoppingToken); + } + await resourcesIndexer.CreateIndexAsync(applicationConfiguration.SearchIndexing.IndexName, stoppingToken); + await resourcesIndexer.PopulateIndexAsync(applicationConfiguration.SearchIndexing.IndexName, applicationConfiguration.SearchIndexing.BatchSize, stoppingToken); + + } + catch (Exception ex) + { + logger.LogError(ex, "Unhandled exception occured"); + } + finally + { + logger.LogInformation("Indexing finished at: {finishTime}", DateTime.Now); + await telemetryClient.FlushAsync(stoppingToken); + } + } +} diff --git a/src/Childrens-Social-Care-CPD-Indexer/appsettings.json b/src/Childrens-Social-Care-CPD-Indexer/appsettings.json index b2dcdb6..96a7301 100644 --- a/src/Childrens-Social-Care-CPD-Indexer/appsettings.json +++ b/src/Childrens-Social-Care-CPD-Indexer/appsettings.json @@ -3,6 +3,13 @@ "LogLevel": { "Default": "Information", "Microsoft.Hosting.Lifetime": "Information" + }, + "ApplicationInsights": { + "LogLevel": { + "Default": "Information", + "Microsoft": "Information", + "Microsoft.Hosting.Lifetime": "Information" + } } } }