From 4c1330dd0cea3a32eafc0f9c9d49e4c0f941d44d Mon Sep 17 00:00:00 2001 From: killij <51908793+killij@users.noreply.github.com> Date: Fri, 8 Dec 2023 15:52:35 +0000 Subject: [PATCH 01/23] fix: Azure throws an Exception when the index doesn't exist Why do companies do this? Just imagine if SQL Server threw an exception every time a query didn't return any results. Not finding something IS NOT EXCEPTIONAL. --- ...ndexerTest.cs => ResourcesIndexerTests.cs} | 41 +++++++++++++++---- .../Core/ResourcesIndexer.cs | 29 ++++++++++--- 2 files changed, 58 insertions(+), 12 deletions(-) rename src/Childrens-Social-Care-CPD-Indexer.Tests/Core/{ResourcesIndexerTest.cs => ResourcesIndexerTests.cs} (79%) diff --git a/src/Childrens-Social-Care-CPD-Indexer.Tests/Core/ResourcesIndexerTest.cs b/src/Childrens-Social-Care-CPD-Indexer.Tests/Core/ResourcesIndexerTests.cs similarity index 79% rename from src/Childrens-Social-Care-CPD-Indexer.Tests/Core/ResourcesIndexerTest.cs rename to src/Childrens-Social-Care-CPD-Indexer.Tests/Core/ResourcesIndexerTests.cs index fd27f3e..199e241 100644 --- a/src/Childrens-Social-Care-CPD-Indexer.Tests/Core/ResourcesIndexerTest.cs +++ b/src/Childrens-Social-Care-CPD-Indexer.Tests/Core/ResourcesIndexerTests.cs @@ -1,13 +1,42 @@ using Azure; +using Azure.Core; using Azure.Search.Documents; using Azure.Search.Documents.Indexes; using Azure.Search.Documents.Indexes.Models; using Childrens_Social_Care_CPD_Indexer.Core; using Microsoft.Extensions.Logging; +using NSubstitute.ExceptionExtensions; +using System.Diagnostics.CodeAnalysis; namespace Childrens_Social_Care_CPD_Indexer.Tests.Core; -public class ResourcesIndexerTest +internal sealed class MockResponse : Response +{ + public override int Status => 404; + public override string ReasonPhrase => string.Empty; + + public override Stream? ContentStream + { + get => new MemoryStream(); + set => throw new NotImplementedException(); + } + public override string ClientRequestId + { + get => throw new NotImplementedException(); + set => throw new NotImplementedException(); + } + + public override void Dispose() => throw new NotImplementedException(); + protected override bool ContainsHeader(string name) => false; + protected override IEnumerable EnumerateHeaders() => Array.Empty(); + protected override bool TryGetHeader(string name, [NotNullWhen(true)] out string? value) { + value = null; + return false; + } + protected override bool TryGetHeaderValues(string name, [NotNullWhen(true)] out IEnumerable? values) => throw new NotImplementedException(); +} + +public class ResourcesIndexerTests { private ILogger _logger; private SearchIndexClient _client; @@ -28,9 +57,8 @@ public void Setup() public async Task DeleteIndexAsync_Skips_Deletion_If_Index_Does_Not_Exist() { // arrange - var response = Substitute.For>(); - response.HasValue.Returns(false); - _client.GetIndexAsync(Arg.Any(), Arg.Any()).Returns(Task.FromResult(response)); + var exception = new RequestFailedException(new MockResponse()); + _client.GetIndexAsync(Arg.Any(), Arg.Any()).Throws(exception); // act await _sut.DeleteIndexAsync("foo"); @@ -77,9 +105,8 @@ public async Task DeleteIndexAsync_Logs_Failure_To_Delete_Index() public async Task CreateIndexAsync_Creates_The_Index() { // arrange - var getIndexResult = Substitute.For>(); - getIndexResult.HasValue.Returns(false); - _client.GetIndexAsync(Arg.Any(), Arg.Any()).Returns(Task.FromResult(getIndexResult)); + var exception = new RequestFailedException(new MockResponse()); + _client.GetIndexAsync(Arg.Any(), Arg.Any()).Throws(exception); SearchIndex? searchIndex = null; await _client.CreateIndexAsync(Arg.Do(x => searchIndex = x), Arg.Any()); diff --git a/src/Childrens-Social-Care-CPD-Indexer/Core/ResourcesIndexer.cs b/src/Childrens-Social-Care-CPD-Indexer/Core/ResourcesIndexer.cs index 1c3124d..d8e6841 100644 --- a/src/Childrens-Social-Care-CPD-Indexer/Core/ResourcesIndexer.cs +++ b/src/Childrens-Social-Care-CPD-Indexer/Core/ResourcesIndexer.cs @@ -1,14 +1,33 @@ -using Azure.Search.Documents.Indexes; +using Azure; +using Azure.Search.Documents.Indexes; using Azure.Search.Documents.Indexes.Models; namespace Childrens_Social_Care_CPD_Indexer.Core; internal class ResourcesIndexer(SearchIndexClient searchIndexClient, IDocumentFetcher documentFetcher, ILogger logger): IResourcesIndexer { + private async Task IndexExistsAsync(string indexName, CancellationToken cancellationToken) + { + try + { + var index = await searchIndexClient.GetIndexAsync(indexName, cancellationToken); + return index.HasValue; + } + catch (RequestFailedException rf) + { + if (rf.Status == 404) + { + return false; + } + + throw; + } + } + public async Task CreateIndexAsync(string indexName, CancellationToken cancellationToken = default) { - var index = await searchIndexClient.GetIndexAsync(indexName, cancellationToken); - if (index.HasValue) + var indexExists = await IndexExistsAsync(indexName, cancellationToken); + if (indexExists) { logger.LogInformation("Index already exists, skipping creation."); return; @@ -25,8 +44,8 @@ public async Task CreateIndexAsync(string indexName, CancellationToken cancellat public async Task DeleteIndexAsync(string indexName, CancellationToken cancellationToken = default) { logger.LogInformation("Deleting index..."); - var index = await searchIndexClient.GetIndexAsync(indexName, cancellationToken); - if (index.HasValue) + var indexExists = await IndexExistsAsync(indexName, cancellationToken); + if (indexExists) { var deleteResponse = await searchIndexClient.DeleteIndexAsync(indexName, cancellationToken); if (deleteResponse.IsError) From 5e24f4e99c3828230e8b9142795c53bb99bffc84 Mon Sep 17 00:00:00 2001 From: cairnsj <51908793+killij@users.noreply.github.com> Date: Mon, 11 Dec 2023 09:57:41 +0000 Subject: [PATCH 02/23] Add telemetry client --- .../Core/ResourcesIndexerTests.cs | 7 ++- .../MockTelemetryClient.cs | 48 +++++++++++++++++++ .../Core/ResourcesIndexer.cs | 12 +++-- .../IndexingService.cs | 2 - .../Program.cs | 23 ++++++--- .../appsettings.json | 5 ++ 6 files changed, 81 insertions(+), 16 deletions(-) create mode 100644 src/Childrens-Social-Care-CPD-Indexer.Tests/MockTelemetryClient.cs diff --git a/src/Childrens-Social-Care-CPD-Indexer.Tests/Core/ResourcesIndexerTests.cs b/src/Childrens-Social-Care-CPD-Indexer.Tests/Core/ResourcesIndexerTests.cs index 199e241..b15d8df 100644 --- a/src/Childrens-Social-Care-CPD-Indexer.Tests/Core/ResourcesIndexerTests.cs +++ b/src/Childrens-Social-Care-CPD-Indexer.Tests/Core/ResourcesIndexerTests.cs @@ -38,7 +38,7 @@ protected override bool TryGetHeader(string name, [NotNullWhen(true)] out string public class ResourcesIndexerTests { - private ILogger _logger; + private ILogger _logger; private SearchIndexClient _client; private IDocumentFetcher _documentFetcher; private ResourcesIndexer _sut; @@ -50,7 +50,7 @@ public void Setup() _client = Substitute.For(); _documentFetcher = Substitute.For(); - _sut = new ResourcesIndexer(_client, _documentFetcher, _logger); + _sut = new ResourcesIndexer(_client, _documentFetcher, _logger, MockTelemetryClient.Create()); } [Test] @@ -168,7 +168,6 @@ public async Task PopulateIndexAsync_Uploads_Documents_In_Multiple_Batches() await _sut.PopulateIndexAsync("foo", 10); - await client.Received(2) - .UploadDocumentsAsync(documents, Arg.Any(), Arg.Any()); + await client.Received(2).UploadDocumentsAsync(documents, Arg.Any(), Arg.Any()); } } \ No newline at end of file diff --git a/src/Childrens-Social-Care-CPD-Indexer.Tests/MockTelemetryClient.cs b/src/Childrens-Social-Care-CPD-Indexer.Tests/MockTelemetryClient.cs new file mode 100644 index 0000000..61bf574 --- /dev/null +++ b/src/Childrens-Social-Care-CPD-Indexer.Tests/MockTelemetryClient.cs @@ -0,0 +1,48 @@ +using Microsoft.ApplicationInsights; +using Microsoft.ApplicationInsights.Channel; +using Microsoft.ApplicationInsights.Extensibility; +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Childrens_Social_Care_CPD_Indexer.Tests; + +public class MockTelemetryChannel : ITelemetryChannel +{ + public ConcurrentBag SentTelemtries = new ConcurrentBag(); + public bool IsFlushed { get; private set; } + public bool? DeveloperMode { get; set; } + public string EndpointAddress { get; set; } + + public void Send(ITelemetry item) + { + this.SentTelemtries.Add(item); + } + + public void Flush() + { + this.IsFlushed = true; + } + + public void Dispose() + { + + } +} + +internal static class MockTelemetryClient +{ + public static TelemetryClient Create() + { + var mockTelemetryConfig = new TelemetryConfiguration + { + TelemetryChannel = new MockTelemetryChannel(), + }; + + return new TelemetryClient(mockTelemetryConfig); + } + +} diff --git a/src/Childrens-Social-Care-CPD-Indexer/Core/ResourcesIndexer.cs b/src/Childrens-Social-Care-CPD-Indexer/Core/ResourcesIndexer.cs index d8e6841..e26f1a6 100644 --- a/src/Childrens-Social-Care-CPD-Indexer/Core/ResourcesIndexer.cs +++ b/src/Childrens-Social-Care-CPD-Indexer/Core/ResourcesIndexer.cs @@ -1,13 +1,16 @@ using Azure; using Azure.Search.Documents.Indexes; using Azure.Search.Documents.Indexes.Models; +using Microsoft.ApplicationInsights; +using Microsoft.ApplicationInsights.DataContracts; namespace Childrens_Social_Care_CPD_Indexer.Core; -internal class ResourcesIndexer(SearchIndexClient searchIndexClient, IDocumentFetcher documentFetcher, ILogger logger): IResourcesIndexer +internal class ResourcesIndexer(SearchIndexClient searchIndexClient, IDocumentFetcher documentFetcher, ILogger logger, TelemetryClient telemetryClient) : IResourcesIndexer { private async Task IndexExistsAsync(string indexName, CancellationToken cancellationToken) { + using var operation = telemetryClient.StartOperation("GetIndexAsync"); try { var index = await searchIndexClient.GetIndexAsync(indexName, cancellationToken); @@ -19,13 +22,14 @@ private async Task IndexExistsAsync(string indexName, CancellationToken ca { return false; } - + telemetryClient.TrackException(rf); throw; } } public async Task CreateIndexAsync(string indexName, CancellationToken cancellationToken = default) { + using var operation = telemetryClient.StartOperation("CreateIndexAsync"); var indexExists = await IndexExistsAsync(indexName, cancellationToken); if (indexExists) { @@ -43,6 +47,7 @@ public async Task CreateIndexAsync(string indexName, CancellationToken cancellat public async Task DeleteIndexAsync(string indexName, CancellationToken cancellationToken = default) { + using var operation = telemetryClient.StartOperation("DeleteIndexAsync"); logger.LogInformation("Deleting index..."); var indexExists = await IndexExistsAsync(indexName, cancellationToken); if (indexExists) @@ -58,9 +63,9 @@ public async Task DeleteIndexAsync(string indexName, CancellationToken cancellat public async Task PopulateIndexAsync(string indexName, int batchSize, CancellationToken cancellationToken = default) { + using var operation = telemetryClient.StartOperation("PopulateIndexAsync"); logger.LogInformation("Populating index..."); var searchClient = searchIndexClient.GetSearchClient(indexName); - var skip = 0; var batch = await documentFetcher.FetchBatchAsync(batchSize, skip, cancellationToken); @@ -70,6 +75,7 @@ public async Task PopulateIndexAsync(string indexName, int batchSize, Cancellati await searchClient.UploadDocumentsAsync(batch, null, cancellationToken); cancellationToken.ThrowIfCancellationRequested(); logger.LogInformation("Uploaded batch to index"); + telemetryClient.TrackEvent("Uploaded batch"); skip += batch.Length; batch = await documentFetcher.FetchBatchAsync(batchSize, skip, cancellationToken); } diff --git a/src/Childrens-Social-Care-CPD-Indexer/IndexingService.cs b/src/Childrens-Social-Care-CPD-Indexer/IndexingService.cs index b6b3a8c..fac8624 100644 --- a/src/Childrens-Social-Care-CPD-Indexer/IndexingService.cs +++ b/src/Childrens-Social-Care-CPD-Indexer/IndexingService.cs @@ -1,5 +1,3 @@ -using Microsoft.ApplicationInsights.DataContracts; -using Microsoft.ApplicationInsights; using Childrens_Social_Care_CPD_Indexer.Core; namespace Childrens_Social_Care_CPD_Indexer; diff --git a/src/Childrens-Social-Care-CPD-Indexer/Program.cs b/src/Childrens-Social-Care-CPD-Indexer/Program.cs index 5dc3f15..4647f81 100644 --- a/src/Childrens-Social-Care-CPD-Indexer/Program.cs +++ b/src/Childrens-Social-Care-CPD-Indexer/Program.cs @@ -1,5 +1,4 @@ using System.Diagnostics.CodeAnalysis; -using Microsoft.ApplicationInsights.WorkerService; using Childrens_Social_Care_CPD_Indexer.Core; using Childrens_Social_Care_CPD_Indexer; using Contentful.Core; @@ -7,21 +6,30 @@ using Contentful.Core.Configuration; using Azure.Search.Documents.Indexes; using Azure; +using Microsoft.ApplicationInsights.WorkerService; +using Microsoft.ApplicationInsights; var builder = Host.CreateDefaultBuilder(args); builder.ConfigureServices((context, services) => { - var config = new ResourcesIndexerConfig(context.Configuration); + // Our configuration services.AddTransient(); + var config = new ResourcesIndexerConfig(context.Configuration); - var options = new ApplicationInsightsServiceOptions + // Service + services.AddHostedService(); + + // Logging + services.AddLogging(); + var options = new ApplicationInsightsServiceOptions() { ApplicationVersion = config.ApplicationVersion, - ConnectionString = config.AppInsightsConnectionString + ConnectionString = config.AppInsightsConnectionString, }; - services.AddApplicationInsightsTelemetryWorkerService(options); + + // Code dependencies services.TryAddTransient(); services.AddTransient(servicesProvider => { var httpClient = servicesProvider.GetRequiredService(); @@ -41,10 +49,11 @@ var documentFetcher = servicesProvider.GetRequiredService(); var searchEndpointUri = new Uri(config.Endpoint); var searchIndexClient = new SearchIndexClient(searchEndpointUri, new AzureKeyCredential(config.ApiKey)); - return new ResourcesIndexer(searchIndexClient, documentFetcher, logger); + var telemtryClient = servicesProvider.GetRequiredService(); + return new ResourcesIndexer(searchIndexClient, documentFetcher, logger, telemtryClient); }); - services.AddHostedService(); + }); using (var host = builder.Build()) diff --git a/src/Childrens-Social-Care-CPD-Indexer/appsettings.json b/src/Childrens-Social-Care-CPD-Indexer/appsettings.json index b2dcdb6..e770633 100644 --- a/src/Childrens-Social-Care-CPD-Indexer/appsettings.json +++ b/src/Childrens-Social-Care-CPD-Indexer/appsettings.json @@ -3,6 +3,11 @@ "LogLevel": { "Default": "Information", "Microsoft.Hosting.Lifetime": "Information" + }, + "ApplicationInsights": { + "LogLevel": { + "Default": "Information" + } } } } From e2f2d64801ca5c4de0274d390e91c8ead2ea98e6 Mon Sep 17 00:00:00 2001 From: killij <51908793+killij@users.noreply.github.com> Date: Mon, 11 Dec 2023 10:09:58 +0000 Subject: [PATCH 03/23] feat: Add telemetry client (#17) * Update Program.cs * Update MockTelemetryClient.cs --- .../MockTelemetryClient.cs | 28 ++++--------------- .../Program.cs | 2 -- 2 files changed, 6 insertions(+), 24 deletions(-) diff --git a/src/Childrens-Social-Care-CPD-Indexer.Tests/MockTelemetryClient.cs b/src/Childrens-Social-Care-CPD-Indexer.Tests/MockTelemetryClient.cs index 61bf574..1eb0a22 100644 --- a/src/Childrens-Social-Care-CPD-Indexer.Tests/MockTelemetryClient.cs +++ b/src/Childrens-Social-Care-CPD-Indexer.Tests/MockTelemetryClient.cs @@ -1,36 +1,20 @@ using Microsoft.ApplicationInsights; using Microsoft.ApplicationInsights.Channel; using Microsoft.ApplicationInsights.Extensibility; -using System; using System.Collections.Concurrent; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; namespace Childrens_Social_Care_CPD_Indexer.Tests; -public class MockTelemetryChannel : ITelemetryChannel +internal sealed class MockTelemetryChannel : ITelemetryChannel { - public ConcurrentBag SentTelemtries = new ConcurrentBag(); + public ConcurrentBag SentTelemtries = []; public bool IsFlushed { get; private set; } public bool? DeveloperMode { get; set; } - public string EndpointAddress { get; set; } + public string EndpointAddress { get; set; } = string.Empty; - public void Send(ITelemetry item) - { - this.SentTelemtries.Add(item); - } - - public void Flush() - { - this.IsFlushed = true; - } - - public void Dispose() - { - - } + public void Send(ITelemetry item) => SentTelemtries.Add(item); + public void Flush() => IsFlushed = true; + public void Dispose() {} } internal static class MockTelemetryClient diff --git a/src/Childrens-Social-Care-CPD-Indexer/Program.cs b/src/Childrens-Social-Care-CPD-Indexer/Program.cs index 4647f81..3a1eb29 100644 --- a/src/Childrens-Social-Care-CPD-Indexer/Program.cs +++ b/src/Childrens-Social-Care-CPD-Indexer/Program.cs @@ -52,8 +52,6 @@ var telemtryClient = servicesProvider.GetRequiredService(); return new ResourcesIndexer(searchIndexClient, documentFetcher, logger, telemtryClient); }); - - }); using (var host = builder.Build()) From 9c334f997638fc2a2dcc377477e98f1477c0364d Mon Sep 17 00:00:00 2001 From: killij <51908793+killij@users.noreply.github.com> Date: Tue, 12 Dec 2023 13:55:42 +0000 Subject: [PATCH 04/23] feat: Switch to using an Azure Function instead of a Container App Job (#18) BREAKING CHANGE: Container App Job -> Azure function --- .github/workflows/build-docker-image.yml | 95 ------- .github/workflows/dotnet-publish.yml | 37 +++ .../IndexingServiceTests.cs | 25 +- src/Childrens-Social-Care-CPD-Indexer.sln | 12 +- .../.gitignore | 264 ++++++++++++++++++ .../Childrens-Social-Care-CPD-Indexer.csproj | 33 ++- .../Core/IResourcesIndexer.cs | 2 +- .../Core/IResourcesIndexerConfig.cs | 2 +- .../Core/ResourcesIndexer.cs | 1 + .../Dockerfile | 11 +- .../{IndexingService.cs => Indexer.cs} | 16 +- .../Program.cs | 100 ++++--- .../Properties/launchSettings.json | 13 +- .../Properties/serviceDependencies.json | 11 + .../Properties/serviceDependencies.local.json | 11 + .../ResourcesIndexerConfig.cs | 1 + .../appsettings.Development.json | 8 - .../appsettings.json | 13 - .../host.json | 12 + 19 files changed, 444 insertions(+), 223 deletions(-) delete mode 100644 .github/workflows/build-docker-image.yml create mode 100644 .github/workflows/dotnet-publish.yml create mode 100644 src/Childrens-Social-Care-CPD-Indexer/.gitignore rename src/Childrens-Social-Care-CPD-Indexer/{IndexingService.cs => Indexer.cs} (66%) create mode 100644 src/Childrens-Social-Care-CPD-Indexer/Properties/serviceDependencies.json create mode 100644 src/Childrens-Social-Care-CPD-Indexer/Properties/serviceDependencies.local.json delete mode 100644 src/Childrens-Social-Care-CPD-Indexer/appsettings.Development.json delete mode 100644 src/Childrens-Social-Care-CPD-Indexer/appsettings.json create mode 100644 src/Childrens-Social-Care-CPD-Indexer/host.json diff --git a/.github/workflows/build-docker-image.yml b/.github/workflows/build-docker-image.yml deleted file mode 100644 index b3b3c08..0000000 --- a/.github/workflows/build-docker-image.yml +++ /dev/null @@ -1,95 +0,0 @@ -name: Docker Publish - -on: - release: - types: [published] - -env: - REGISTRY: ghcr.io - IMAGE_NAME: ${{ github.repository }} - RELEASE_TAG: ${{ github.event.release.tag_name }} - -jobs: - build: - name: 'Docker Publish' - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - # This is used to complete the identity challenge with sigstore/fulcio when running outside of PRs. - id-token: write - - steps: - # Checkout the release tag version - - name: Checkout repository ${{ env.RELEASE_TAG }} - uses: actions/checkout@v3 - with: - ref: ${{ env.RELEASE_TAG }} - - # Get git commit hash - - name: Get short hash - run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_ENV - - # Need to lower case the image name for the docker tags when publishing - - name: Downcase IMAGE_NAME variable - run: echo "IMAGE_NAME_LOWER=${IMAGE_NAME,,}" >> $GITHUB_ENV - - # Sort out the image tags - - name: Set initial tag - run: echo "IMAGE_TAGS=${{ env.REGISTRY }}/${{ env.IMAGE_NAME_LOWER }}:${{ env.RELEASE_TAG }}" >> $GITHUB_ENV - - - name: Add latest tag if we're not production release - if: contains(env.RELEASE_TAG, 'next') - run: echo "IMAGE_TAGS=${{ env.IMAGE_TAGS }},${{ env.REGISTRY }}/${{ env.IMAGE_NAME_LOWER }}:latest" >> $GITHUB_ENV - - #debug - - name: Log the tags - run: echo "Calculated tags value => ${{ env.IMAGE_TAGS }}" - - # Setup docker build tool - - name: Setup Docker buildx - uses: docker/setup-buildx-action@v3 - - # Login against a Docker registry - - name: Log into registry ${{ env.REGISTRY }} - uses: docker/login-action@v3 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - # Extract metadata (tags, labels) for Docker - - name: Extract Docker metadata - id: meta - uses: docker/metadata-action@v5 - with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - - # Build and push Docker image with Buildx (don't push on PR) - - name: Build and push Docker image - id: build-and-push - uses: docker/build-push-action@v5 - with: - context: ./src - file: ./src/Childrens-Social-Care-CPD-Indexer/Dockerfile - push: true - tags: ${{ env.IMAGE_TAGS }} - labels: ${{ steps.meta.outputs.labels }} - build-args: | - VCSREF=${{ env.sha_short }} - VCSTAG=${{ env.RELEASE_TAG }} - cache-from: type=gha - cache-to: type=gha,mode=max - - # Sign the resulting Docker image digest. - # This will only write to the public Rekor transparency log when the Docker - # repository is public to avoid leaking data. If you would like to publish - # transparency data even for private images, pass --force to cosign below. - # https://github.com/sigstore/cosign - - name: Install Cosign - uses: sigstore/cosign-installer@v3.2.0 - - name: Check install! - run: cosign version - - name: Sign the published Docker image - # This step uses the identity token to provision an ephemeral certificate against the sigstore community Fulcio instance. - run: echo "${{ steps.meta.outputs.tags }}" | xargs -I {} cosign sign --yes {}@${{ steps.build-and-push.outputs.digest }} diff --git a/.github/workflows/dotnet-publish.yml b/.github/workflows/dotnet-publish.yml new file mode 100644 index 0000000..8967a03 --- /dev/null +++ b/.github/workflows/dotnet-publish.yml @@ -0,0 +1,37 @@ +name: Dotnet Publish + +on: + release: + types: [published] + +env: + RELEASE_TAG: ${{ github.event.release.tag_name }} + +jobs: + build: + name: 'Build Deployment Package' + runs-on: ubuntu-latest + steps: + - name: Checkout repository ${{ env.RELEASE_TAG }} + uses: actions/checkout@v3 + with: + ref: ${{ env.RELEASE_TAG }} + + - name: Setup dotnet + uses: actions/setup-dotnet@v3 + with: + dotnet-version: '8.x' + + - name: dotnet publish + run: dotnet publish ./Childrens-Social-Care-CPD-Indexer/Childrens-Social-Care-CPD-Indexer.csproj -c Release -o ./publish + working-directory: ./src + + - name: Zip output + run: zip -r ../${{ env.RELEASE_TAG }}.zip ./ + working-directory: ./src/publish + + - name: Upload artifacts + uses: actions/upload-artifact@v3 + with: + name: resource-indexing-deployment + path: ./src/${{ env.RELEASE_TAG }}.zip \ No newline at end of file diff --git a/src/Childrens-Social-Care-CPD-Indexer.Tests/IndexingServiceTests.cs b/src/Childrens-Social-Care-CPD-Indexer.Tests/IndexingServiceTests.cs index 57280f6..782e78a 100644 --- a/src/Childrens-Social-Care-CPD-Indexer.Tests/IndexingServiceTests.cs +++ b/src/Childrens-Social-Care-CPD-Indexer.Tests/IndexingServiceTests.cs @@ -1,5 +1,6 @@ using Childrens_Social_Care_CPD_Indexer.Core; using Microsoft.ApplicationInsights; +using Microsoft.Azure.Functions.Worker; using Microsoft.Extensions.Logging; using NSubstitute.ExceptionExtensions; @@ -7,28 +8,18 @@ namespace Childrens_Social_Care_CPD_Indexer.Tests; public class IndexingServiceTests { - private ILogger _logger; + private ILogger _logger; private IResourcesIndexerConfig _config; private IResourcesIndexer _indexer; - private IndexingService _sut; + private Indexer _sut; [SetUp] public void Setup() { - _logger = Substitute.For>(); + _logger = Substitute.For>(); _config = Substitute.For(); _indexer = Substitute.For(); - _sut = new IndexingService(_indexer, _logger, _config); - } - - [Test] - public void StopAsync_Returns_Completed_Task() - { - // act - var task = _sut.StopAsync(default); - - // assert - task.IsCompleted.Should().BeTrue(); + _sut = new Indexer(_logger, _indexer, _config); } [Test] @@ -38,7 +29,7 @@ public async Task StartAsync_Deletes_Index_If_Configured() _config.RecreateIndex.Returns(true); // act - await _sut.StartAsync(default); + await _sut.Run(new TimerInfo()); // assert await _indexer.Received(1).DeleteIndexAsync(Arg.Any(), Arg.Any()); @@ -52,7 +43,7 @@ public async Task StartAsync_Populates_Index() _config.RecreateIndex.Returns(false); // act - await _sut.StartAsync(default); + await _sut.Run(new TimerInfo()); // assert await _indexer.Received(1).PopulateIndexAsync(Arg.Any(), Arg.Any(), Arg.Any()); @@ -67,7 +58,7 @@ public async Task StartAsync_Logs_Exception() _indexer.DeleteIndexAsync(Arg.Any(), Arg.Any()).Throws(exception); // act - await _sut.StartAsync(default); + await _sut.Run(new TimerInfo()); // assert _logger.Received(1).LogError(exception, "Unhandled exception occured"); diff --git a/src/Childrens-Social-Care-CPD-Indexer.sln b/src/Childrens-Social-Care-CPD-Indexer.sln index ede56d3..21908a1 100644 --- a/src/Childrens-Social-Care-CPD-Indexer.sln +++ b/src/Childrens-Social-Care-CPD-Indexer.sln @@ -3,9 +3,9 @@ Microsoft Visual Studio Solution File, Format Version 12.00 # Visual Studio Version 17 VisualStudioVersion = 17.8.34322.80 MinimumVisualStudioVersion = 10.0.40219.1 -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Childrens-Social-Care-CPD-Indexer", "Childrens-Social-Care-CPD-Indexer\Childrens-Social-Care-CPD-Indexer.csproj", "{D806CCEA-5505-4CD9-98B6-08A5F161C06A}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Childrens-Social-Care-CPD-Indexer.Tests", "Childrens-Social-Care-CPD-Indexer.Tests\Childrens-Social-Care-CPD-Indexer.Tests.csproj", "{6984BF56-808E-4294-949D-FFE4B02CCE16}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Childrens-Social-Care-CPD-Indexer.Tests", "Childrens-Social-Care-CPD-Indexer.Tests\Childrens-Social-Care-CPD-Indexer.Tests.csproj", "{6984BF56-808E-4294-949D-FFE4B02CCE16}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Childrens-Social-Care-CPD-Indexer", "Childrens-Social-Care-CPD-Indexer\Childrens-Social-Care-CPD-Indexer.csproj", "{3555944D-1913-4979-B5BF-991C7064613E}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -13,14 +13,14 @@ Global Release|Any CPU = Release|Any CPU EndGlobalSection GlobalSection(ProjectConfigurationPlatforms) = postSolution - {D806CCEA-5505-4CD9-98B6-08A5F161C06A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {D806CCEA-5505-4CD9-98B6-08A5F161C06A}.Debug|Any CPU.Build.0 = Debug|Any CPU - {D806CCEA-5505-4CD9-98B6-08A5F161C06A}.Release|Any CPU.ActiveCfg = Release|Any CPU - {D806CCEA-5505-4CD9-98B6-08A5F161C06A}.Release|Any CPU.Build.0 = Release|Any CPU {6984BF56-808E-4294-949D-FFE4B02CCE16}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {6984BF56-808E-4294-949D-FFE4B02CCE16}.Debug|Any CPU.Build.0 = Debug|Any CPU {6984BF56-808E-4294-949D-FFE4B02CCE16}.Release|Any CPU.ActiveCfg = Release|Any CPU {6984BF56-808E-4294-949D-FFE4B02CCE16}.Release|Any CPU.Build.0 = Release|Any CPU + {3555944D-1913-4979-B5BF-991C7064613E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {3555944D-1913-4979-B5BF-991C7064613E}.Debug|Any CPU.Build.0 = Debug|Any CPU + {3555944D-1913-4979-B5BF-991C7064613E}.Release|Any CPU.ActiveCfg = Release|Any CPU + {3555944D-1913-4979-B5BF-991C7064613E}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/src/Childrens-Social-Care-CPD-Indexer/.gitignore b/src/Childrens-Social-Care-CPD-Indexer/.gitignore new file mode 100644 index 0000000..ff5b00c --- /dev/null +++ b/src/Childrens-Social-Care-CPD-Indexer/.gitignore @@ -0,0 +1,264 @@ +## Ignore Visual Studio temporary files, build results, and +## files generated by popular Visual Studio add-ons. + +# Azure Functions localsettings file +local.settings.json + +# User-specific files +*.suo +*.user +*.userosscache +*.sln.docstates + +# User-specific files (MonoDevelop/Xamarin Studio) +*.userprefs + +# Build results +[Dd]ebug/ +[Dd]ebugPublic/ +[Rr]elease/ +[Rr]eleases/ +x64/ +x86/ +bld/ +[Bb]in/ +[Oo]bj/ +[Ll]og/ + +# Visual Studio 2015 cache/options directory +.vs/ +# Uncomment if you have tasks that create the project's static files in wwwroot +#wwwroot/ + +# MSTest test Results +[Tt]est[Rr]esult*/ +[Bb]uild[Ll]og.* + +# NUNIT +*.VisualState.xml +TestResult.xml + +# Build Results of an ATL Project +[Dd]ebugPS/ +[Rr]eleasePS/ +dlldata.c + +# DNX +project.lock.json +project.fragment.lock.json +artifacts/ + +*_i.c +*_p.c +*_i.h +*.ilk +*.meta +*.obj +*.pch +*.pdb +*.pgc +*.pgd +*.rsp +*.sbr +*.tlb +*.tli +*.tlh +*.tmp +*.tmp_proj +*.log +*.vspscc +*.vssscc +.builds +*.pidb +*.svclog +*.scc + +# Chutzpah Test files +_Chutzpah* + +# Visual C++ cache files +ipch/ +*.aps +*.ncb +*.opendb +*.opensdf +*.sdf +*.cachefile +*.VC.db +*.VC.VC.opendb + +# Visual Studio profiler +*.psess +*.vsp +*.vspx +*.sap + +# TFS 2012 Local Workspace +$tf/ + +# Guidance Automation Toolkit +*.gpState + +# ReSharper is a .NET coding add-in +_ReSharper*/ +*.[Rr]e[Ss]harper +*.DotSettings.user + +# JustCode is a .NET coding add-in +.JustCode + +# TeamCity is a build add-in +_TeamCity* + +# DotCover is a Code Coverage Tool +*.dotCover + +# NCrunch +_NCrunch_* +.*crunch*.local.xml +nCrunchTemp_* + +# MightyMoose +*.mm.* +AutoTest.Net/ + +# Web workbench (sass) +.sass-cache/ + +# Installshield output folder +[Ee]xpress/ + +# DocProject is a documentation generator add-in +DocProject/buildhelp/ +DocProject/Help/*.HxT +DocProject/Help/*.HxC +DocProject/Help/*.hhc +DocProject/Help/*.hhk +DocProject/Help/*.hhp +DocProject/Help/Html2 +DocProject/Help/html + +# Click-Once directory +publish/ + +# Publish Web Output +*.[Pp]ublish.xml +*.azurePubxml +# TODO: Comment the next line if you want to checkin your web deploy settings +# but database connection strings (with potential passwords) will be unencrypted +#*.pubxml +*.publishproj + +# Microsoft Azure Web App publish settings. Comment the next line if you want to +# checkin your Azure Web App publish settings, but sensitive information contained +# in these scripts will be unencrypted +PublishScripts/ + +# NuGet Packages +*.nupkg +# The packages folder can be ignored because of Package Restore +**/packages/* +# except build/, which is used as an MSBuild target. +!**/packages/build/ +# Uncomment if necessary however generally it will be regenerated when needed +#!**/packages/repositories.config +# NuGet v3's project.json files produces more ignoreable files +*.nuget.props +*.nuget.targets + +# Microsoft Azure Build Output +csx/ +*.build.csdef + +# Microsoft Azure Emulator +ecf/ +rcf/ + +# Windows Store app package directories and files +AppPackages/ +BundleArtifacts/ +Package.StoreAssociation.xml +_pkginfo.txt + +# Visual Studio cache files +# files ending in .cache can be ignored +*.[Cc]ache +# but keep track of directories ending in .cache +!*.[Cc]ache/ + +# Others +ClientBin/ +~$* +*~ +*.dbmdl +*.dbproj.schemaview +*.jfm +*.pfx +*.publishsettings +node_modules/ +orleans.codegen.cs + +# Since there are multiple workflows, uncomment next line to ignore bower_components +# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) +#bower_components/ + +# RIA/Silverlight projects +Generated_Code/ + +# Backup & report files from converting an old project file +# to a newer Visual Studio version. Backup files are not needed, +# because we have git ;-) +_UpgradeReport_Files/ +Backup*/ +UpgradeLog*.XML +UpgradeLog*.htm + +# SQL Server files +*.mdf +*.ldf + +# Business Intelligence projects +*.rdl.data +*.bim.layout +*.bim_*.settings + +# Microsoft Fakes +FakesAssemblies/ + +# GhostDoc plugin setting file +*.GhostDoc.xml + +# Node.js Tools for Visual Studio +.ntvs_analysis.dat + +# Visual Studio 6 build log +*.plg + +# Visual Studio 6 workspace options file +*.opt + +# Visual Studio LightSwitch build output +**/*.HTMLClient/GeneratedArtifacts +**/*.DesktopClient/GeneratedArtifacts +**/*.DesktopClient/ModelManifest.xml +**/*.Server/GeneratedArtifacts +**/*.Server/ModelManifest.xml +_Pvt_Extensions + +# Paket dependency manager +.paket/paket.exe +paket-files/ + +# FAKE - F# Make +.fake/ + +# JetBrains Rider +.idea/ +*.sln.iml + +# CodeRush +.cr/ + +# Python Tools for Visual Studio (PTVS) +__pycache__/ +*.pyc \ No newline at end of file diff --git a/src/Childrens-Social-Care-CPD-Indexer/Childrens-Social-Care-CPD-Indexer.csproj b/src/Childrens-Social-Care-CPD-Indexer/Childrens-Social-Care-CPD-Indexer.csproj index 48ae2fb..1692839 100644 --- a/src/Childrens-Social-Care-CPD-Indexer/Childrens-Social-Care-CPD-Indexer.csproj +++ b/src/Childrens-Social-Care-CPD-Indexer/Childrens-Social-Care-CPD-Indexer.csproj @@ -1,26 +1,39 @@ - - + net8.0 - enable + v4 + Exe enable - dotnet-Childrens_Social_Care_CPD_Indexer-5181f816-6a9f-4c7a-a524-576a7d37d013 + enable Childrens_Social_Care_CPD_Indexer + /home/site/wwwroot Linux - - - - + + + + - + - + + + PreserveNewest + + + PreserveNewest + Never + + + + + + \ No newline at end of file diff --git a/src/Childrens-Social-Care-CPD-Indexer/Core/IResourcesIndexer.cs b/src/Childrens-Social-Care-CPD-Indexer/Core/IResourcesIndexer.cs index 13c2707..f519948 100644 --- a/src/Childrens-Social-Care-CPD-Indexer/Core/IResourcesIndexer.cs +++ b/src/Childrens-Social-Care-CPD-Indexer/Core/IResourcesIndexer.cs @@ -1,6 +1,6 @@ namespace Childrens_Social_Care_CPD_Indexer.Core; -internal interface IResourcesIndexer +public interface IResourcesIndexer { Task CreateIndexAsync(string indexName, CancellationToken cancellationToken); Task DeleteIndexAsync(string indexName, CancellationToken cancellationToken); diff --git a/src/Childrens-Social-Care-CPD-Indexer/Core/IResourcesIndexerConfig.cs b/src/Childrens-Social-Care-CPD-Indexer/Core/IResourcesIndexerConfig.cs index 9f59606..b83e033 100644 --- a/src/Childrens-Social-Care-CPD-Indexer/Core/IResourcesIndexerConfig.cs +++ b/src/Childrens-Social-Care-CPD-Indexer/Core/IResourcesIndexerConfig.cs @@ -1,6 +1,6 @@ namespace Childrens_Social_Care_CPD_Indexer.Core; -internal interface IResourcesIndexerConfig +public interface IResourcesIndexerConfig { string ApiKey { get; } string AppInsightsConnectionString { get; } diff --git a/src/Childrens-Social-Care-CPD-Indexer/Core/ResourcesIndexer.cs b/src/Childrens-Social-Care-CPD-Indexer/Core/ResourcesIndexer.cs index e26f1a6..408acf7 100644 --- a/src/Childrens-Social-Care-CPD-Indexer/Core/ResourcesIndexer.cs +++ b/src/Childrens-Social-Care-CPD-Indexer/Core/ResourcesIndexer.cs @@ -3,6 +3,7 @@ using Azure.Search.Documents.Indexes.Models; using Microsoft.ApplicationInsights; using Microsoft.ApplicationInsights.DataContracts; +using Microsoft.Extensions.Logging; namespace Childrens_Social_Care_CPD_Indexer.Core; diff --git a/src/Childrens-Social-Care-CPD-Indexer/Dockerfile b/src/Childrens-Social-Care-CPD-Indexer/Dockerfile index 9b63d70..c4a2687 100644 --- a/src/Childrens-Social-Care-CPD-Indexer/Dockerfile +++ b/src/Childrens-Social-Care-CPD-Indexer/Dockerfile @@ -1,8 +1,8 @@ #See https://aka.ms/customizecontainer to learn how to customize your debug container and how Visual Studio uses this Dockerfile to build your images for faster debugging. -FROM mcr.microsoft.com/dotnet/runtime:8.0 AS base -USER app -WORKDIR /app +FROM mcr.microsoft.com/azure-functions/dotnet-isolated:4-dotnet-isolated8.0 AS base +WORKDIR /home/site/wwwroot +EXPOSE 8080 FROM mcr.microsoft.com/dotnet/sdk:8.0 AS build ARG BUILD_CONFIGURATION=Release @@ -18,6 +18,7 @@ ARG BUILD_CONFIGURATION=Release RUN dotnet publish "./Childrens-Social-Care-CPD-Indexer.csproj" -c $BUILD_CONFIGURATION -o /app/publish /p:UseAppHost=false FROM base AS final -WORKDIR /app +WORKDIR /home/site/wwwroot COPY --from=publish /app/publish . -ENTRYPOINT ["dotnet", "Childrens-Social-Care-CPD-Indexer.dll"] \ No newline at end of file +ENV AzureWebJobsScriptRoot=/home/site/wwwroot \ + AzureFunctionsJobHost__Logging__Console__IsEnabled=true \ No newline at end of file diff --git a/src/Childrens-Social-Care-CPD-Indexer/IndexingService.cs b/src/Childrens-Social-Care-CPD-Indexer/Indexer.cs similarity index 66% rename from src/Childrens-Social-Care-CPD-Indexer/IndexingService.cs rename to src/Childrens-Social-Care-CPD-Indexer/Indexer.cs index fac8624..b7c069d 100644 --- a/src/Childrens-Social-Care-CPD-Indexer/IndexingService.cs +++ b/src/Childrens-Social-Care-CPD-Indexer/Indexer.cs @@ -1,10 +1,17 @@ using Childrens_Social_Care_CPD_Indexer.Core; +using Microsoft.Azure.Functions.Worker; +using Microsoft.Extensions.Logging; namespace Childrens_Social_Care_CPD_Indexer; -internal class IndexingService(IResourcesIndexer resourcesIndexer, ILogger logger, IResourcesIndexerConfig config) : IHostedService +public class Indexer(ILogger logger, IResourcesIndexer resourcesIndexer, IResourcesIndexerConfig config) { - public async Task StartAsync(CancellationToken cancellationToken) + [Function("IndexResources")] + public async Task Run([TimerTrigger("0 0 * * SUN" + #if DEBUG + , RunOnStartup= true + #endif + )] TimerInfo myTimer, CancellationToken cancellationToken = default) { logger.LogInformation("Indexing started at: {startTime}", DateTime.Now); try @@ -27,9 +34,4 @@ public async Task StartAsync(CancellationToken cancellationToken) logger.LogInformation("Indexing finished at: {finishTime}", DateTime.Now); } } - - public Task StopAsync(CancellationToken cancellationToken) - { - return Task.CompletedTask; - } } diff --git a/src/Childrens-Social-Care-CPD-Indexer/Program.cs b/src/Childrens-Social-Care-CPD-Indexer/Program.cs index 3a1eb29..f3352b8 100644 --- a/src/Childrens-Social-Care-CPD-Indexer/Program.cs +++ b/src/Childrens-Social-Care-CPD-Indexer/Program.cs @@ -1,65 +1,59 @@ -using System.Diagnostics.CodeAnalysis; using Childrens_Social_Care_CPD_Indexer.Core; using Childrens_Social_Care_CPD_Indexer; -using Contentful.Core; -using Microsoft.Extensions.DependencyInjection.Extensions; -using Contentful.Core.Configuration; +using Microsoft.Azure.Functions.Worker; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.ApplicationInsights.WorkerService; using Azure.Search.Documents.Indexes; using Azure; -using Microsoft.ApplicationInsights.WorkerService; +using Contentful.Core.Configuration; +using Contentful.Core; using Microsoft.ApplicationInsights; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Logging; -var builder = Host.CreateDefaultBuilder(args); +var host = new HostBuilder() + .ConfigureFunctionsWorkerDefaults() + .ConfigureServices((context, services) => + { + services.AddTransient(); + var config = new ResourcesIndexerConfig(context.Configuration); -builder.ConfigureServices((context, services) => -{ - // Our configuration - services.AddTransient(); - var config = new ResourcesIndexerConfig(context.Configuration); - // Service - services.AddHostedService(); - - // Logging - services.AddLogging(); - var options = new ApplicationInsightsServiceOptions() - { - ApplicationVersion = config.ApplicationVersion, - ConnectionString = config.AppInsightsConnectionString, - }; - services.AddApplicationInsightsTelemetryWorkerService(options); - - // Code dependencies - services.TryAddTransient(); - services.AddTransient(servicesProvider => { - var httpClient = servicesProvider.GetRequiredService(); - var resourcesIndexerConfig = servicesProvider.GetRequiredService(); - var contentfulOptions = new ContentfulOptions() + // Logging + var options = new ApplicationInsightsServiceOptions() { - DeliveryApiKey = resourcesIndexerConfig.ContentfulApiKey, - SpaceId = resourcesIndexerConfig.ContentfulSpaceId, - Environment = resourcesIndexerConfig.ContentfulEnvironmentId + ApplicationVersion = config.ApplicationVersion, + ConnectionString = config.AppInsightsConnectionString, }; - return new ContentfulClient(httpClient, contentfulOptions); - }); - services.AddTransient(); - services.AddTransient(servicesProvider => { - var logger = servicesProvider.GetRequiredService>(); - var config = servicesProvider.GetRequiredService(); - var documentFetcher = servicesProvider.GetRequiredService(); - var searchEndpointUri = new Uri(config.Endpoint); - var searchIndexClient = new SearchIndexClient(searchEndpointUri, new AzureKeyCredential(config.ApiKey)); - var telemtryClient = servicesProvider.GetRequiredService(); - return new ResourcesIndexer(searchIndexClient, documentFetcher, logger, telemtryClient); - }); -}); -using (var host = builder.Build()) -{ - var lifetime = host.Services.GetRequiredService(); - await host.StartAsync().ContinueWith(x => lifetime.StopApplication()); -} + services.AddApplicationInsightsTelemetryWorkerService(options); + services.ConfigureFunctionsApplicationInsights(); + + // Code dependencies + services.TryAddTransient(); + services.AddTransient(servicesProvider => { + var httpClient = servicesProvider.GetRequiredService(); + var resourcesIndexerConfig = servicesProvider.GetRequiredService(); + var contentfulOptions = new ContentfulOptions() + { + DeliveryApiKey = resourcesIndexerConfig.ContentfulApiKey, + SpaceId = resourcesIndexerConfig.ContentfulSpaceId, + Environment = resourcesIndexerConfig.ContentfulEnvironmentId + }; + return new ContentfulClient(httpClient, contentfulOptions); + }); + services.AddTransient(); + services.AddTransient(servicesProvider => { + var logger = servicesProvider.GetRequiredService>(); + var config = servicesProvider.GetRequiredService(); + var documentFetcher = servicesProvider.GetRequiredService(); + var searchEndpointUri = new Uri(config.Endpoint); + var searchIndexClient = new SearchIndexClient(searchEndpointUri, new AzureKeyCredential(config.ApiKey)); + var telemtryClient = servicesProvider.GetRequiredService(); + return new ResourcesIndexer(searchIndexClient, documentFetcher, logger, telemtryClient); + }); + }) + .Build(); -[ExcludeFromCodeCoverage] -public partial class Program() -{} \ No newline at end of file +host.Run(); diff --git a/src/Childrens-Social-Care-CPD-Indexer/Properties/launchSettings.json b/src/Childrens-Social-Care-CPD-Indexer/Properties/launchSettings.json index cae296e..3939375 100644 --- a/src/Childrens-Social-Care-CPD-Indexer/Properties/launchSettings.json +++ b/src/Childrens-Social-Care-CPD-Indexer/Properties/launchSettings.json @@ -2,14 +2,13 @@ "profiles": { "Childrens_Social_Care_CPD_Indexer": { "commandName": "Project", - "environmentVariables": { - "DOTNET_ENVIRONMENT": "Development" - }, - "dotnetRunMessages": true + "commandLineArgs": "--port 7016" }, "Docker": { - "commandName": "Docker" + "commandName": "Docker", + "launchUrl": "{Scheme}://{ServiceHost}:{ServicePort}", + "httpPort": 31453, + "useSSL": false } - }, - "$schema": "http://json.schemastore.org/launchsettings.json" + } } \ No newline at end of file diff --git a/src/Childrens-Social-Care-CPD-Indexer/Properties/serviceDependencies.json b/src/Childrens-Social-Care-CPD-Indexer/Properties/serviceDependencies.json new file mode 100644 index 0000000..df4dcc9 --- /dev/null +++ b/src/Childrens-Social-Care-CPD-Indexer/Properties/serviceDependencies.json @@ -0,0 +1,11 @@ +{ + "dependencies": { + "appInsights1": { + "type": "appInsights" + }, + "storage1": { + "type": "storage", + "connectionId": "AzureWebJobsStorage" + } + } +} \ No newline at end of file diff --git a/src/Childrens-Social-Care-CPD-Indexer/Properties/serviceDependencies.local.json b/src/Childrens-Social-Care-CPD-Indexer/Properties/serviceDependencies.local.json new file mode 100644 index 0000000..b804a28 --- /dev/null +++ b/src/Childrens-Social-Care-CPD-Indexer/Properties/serviceDependencies.local.json @@ -0,0 +1,11 @@ +{ + "dependencies": { + "appInsights1": { + "type": "appInsights.sdk" + }, + "storage1": { + "type": "storage.emulator", + "connectionId": "AzureWebJobsStorage" + } + } +} \ No newline at end of file diff --git a/src/Childrens-Social-Care-CPD-Indexer/ResourcesIndexerConfig.cs b/src/Childrens-Social-Care-CPD-Indexer/ResourcesIndexerConfig.cs index d67a842..1c74e9e 100644 --- a/src/Childrens-Social-Care-CPD-Indexer/ResourcesIndexerConfig.cs +++ b/src/Childrens-Social-Care-CPD-Indexer/ResourcesIndexerConfig.cs @@ -1,4 +1,5 @@ using Childrens_Social_Care_CPD_Indexer.Core; +using Microsoft.Extensions.Configuration; namespace Childrens_Social_Care_CPD_Indexer; diff --git a/src/Childrens-Social-Care-CPD-Indexer/appsettings.Development.json b/src/Childrens-Social-Care-CPD-Indexer/appsettings.Development.json deleted file mode 100644 index b2dcdb6..0000000 --- a/src/Childrens-Social-Care-CPD-Indexer/appsettings.Development.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "Logging": { - "LogLevel": { - "Default": "Information", - "Microsoft.Hosting.Lifetime": "Information" - } - } -} diff --git a/src/Childrens-Social-Care-CPD-Indexer/appsettings.json b/src/Childrens-Social-Care-CPD-Indexer/appsettings.json deleted file mode 100644 index e770633..0000000 --- a/src/Childrens-Social-Care-CPD-Indexer/appsettings.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "Logging": { - "LogLevel": { - "Default": "Information", - "Microsoft.Hosting.Lifetime": "Information" - }, - "ApplicationInsights": { - "LogLevel": { - "Default": "Information" - } - } - } -} diff --git a/src/Childrens-Social-Care-CPD-Indexer/host.json b/src/Childrens-Social-Care-CPD-Indexer/host.json new file mode 100644 index 0000000..ee5cf5f --- /dev/null +++ b/src/Childrens-Social-Care-CPD-Indexer/host.json @@ -0,0 +1,12 @@ +{ + "version": "2.0", + "logging": { + "applicationInsights": { + "samplingSettings": { + "isEnabled": true, + "excludedTypes": "Request" + }, + "enableLiveMetricsFilters": true + } + } +} \ No newline at end of file From 4e8ecce4230ed493a1ae1baa34ef4947123f3865 Mon Sep 17 00:00:00 2001 From: Al West Date: Tue, 9 Jan 2024 10:19:18 +0000 Subject: [PATCH 05/23] Create function-app-deploy.yml Create a new action to deploy the function app to dev --- .github/workflows/function-app-deploy.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 .github/workflows/function-app-deploy.yml diff --git a/.github/workflows/function-app-deploy.yml b/.github/workflows/function-app-deploy.yml new file mode 100644 index 0000000..a91954f --- /dev/null +++ b/.github/workflows/function-app-deploy.yml @@ -0,0 +1,18 @@ +name: Function App Deploy + +on: + release: + types: [published] + +jobs: + build: + name: 'Function App Deploy' + runs-on: ubuntu-latest + steps: + - name: Sign in to Azure + uses: azure/login@v1 + with: + creds: '{"clientId":"${{ secrets.FAD_CLIENT_ID }}","clientSecret":"${{ secrets.FAD_CLIENT_SECRET }}","subscriptionId":"${{ secrets.FAD_SUBSCRIPTION_ID }}","tenantId":"${{ secrets.FAD_TENANT_ID }}"}' + + - name: Deploy Function App (Dev) + run: az functionapp deployment source config --branch main --manual-integration --name s185d01-search --repo-url https://github.com/DFE-Digital/childrens-social-care-cpd-indexer --resource-group s185d01-childrens-social-care-cpd-rg From 24a2dfaa97b7859901418eaa3afb381d86f75819 Mon Sep 17 00:00:00 2001 From: Al West Date: Tue, 9 Jan 2024 11:23:42 +0000 Subject: [PATCH 06/23] Update function-app-deploy.yml Parametrised call to az cli --- .github/workflows/function-app-deploy.yml | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/.github/workflows/function-app-deploy.yml b/.github/workflows/function-app-deploy.yml index a91954f..3c405aa 100644 --- a/.github/workflows/function-app-deploy.yml +++ b/.github/workflows/function-app-deploy.yml @@ -3,7 +3,19 @@ name: Function App Deploy on: release: types: [published] - + inputs: + workspace: + description: 'Environment to deploy to' + required: true + default: 'Dev' + type: choice + options: + - Dev + - Test + - Pre-Prod + - Prod + - Load-Test + jobs: build: name: 'Function App Deploy' @@ -14,5 +26,5 @@ jobs: with: creds: '{"clientId":"${{ secrets.FAD_CLIENT_ID }}","clientSecret":"${{ secrets.FAD_CLIENT_SECRET }}","subscriptionId":"${{ secrets.FAD_SUBSCRIPTION_ID }}","tenantId":"${{ secrets.FAD_TENANT_ID }}"}' - - name: Deploy Function App (Dev) - run: az functionapp deployment source config --branch main --manual-integration --name s185d01-search --repo-url https://github.com/DFE-Digital/childrens-social-care-cpd-indexer --resource-group s185d01-childrens-social-care-cpd-rg + - name: Deploy Function App + run: az functionapp deployment source config --branch main --manual-integration --name ${{ vars.FAD_FUNCTION_APP_NAME }} --repo-url ${{ github.event.repository.name }} --resource-group ${{ vars.FAD_RESOURCE_GROUP }} From e16076842b1073304c86725fa6b277cf294d01c7 Mon Sep 17 00:00:00 2001 From: Al West Date: Tue, 9 Jan 2024 11:59:47 +0000 Subject: [PATCH 07/23] Switch to manual and add tag to deploy --- .github/workflows/function-app-deploy.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/function-app-deploy.yml b/.github/workflows/function-app-deploy.yml index 3c405aa..e3261ee 100644 --- a/.github/workflows/function-app-deploy.yml +++ b/.github/workflows/function-app-deploy.yml @@ -1,9 +1,12 @@ name: Function App Deploy on: - release: - types: [published] + workflow_dispatch: inputs: + tag: + description: 'App Tag to deploy' + required: true + default: 'v1.0.0' workspace: description: 'Environment to deploy to' required: true From 146af22615f4d8c08c9bef7e7a2240a46681f56d Mon Sep 17 00:00:00 2001 From: Al West Date: Tue, 9 Jan 2024 12:03:13 +0000 Subject: [PATCH 08/23] Select environment to use --- .github/workflows/function-app-deploy.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/function-app-deploy.yml b/.github/workflows/function-app-deploy.yml index e3261ee..863bb2d 100644 --- a/.github/workflows/function-app-deploy.yml +++ b/.github/workflows/function-app-deploy.yml @@ -23,6 +23,7 @@ jobs: build: name: 'Function App Deploy' runs-on: ubuntu-latest + environment: ${{ inputs.workspace }} steps: - name: Sign in to Azure uses: azure/login@v1 From e89bba28f6a66a781cbdf1e975288c1139716d1f Mon Sep 17 00:00:00 2001 From: Al West Date: Tue, 9 Jan 2024 16:14:31 +0000 Subject: [PATCH 09/23] Use wget to fetch asset --- .github/workflows/function-app-deploy.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/function-app-deploy.yml b/.github/workflows/function-app-deploy.yml index 863bb2d..7111e6a 100644 --- a/.github/workflows/function-app-deploy.yml +++ b/.github/workflows/function-app-deploy.yml @@ -30,5 +30,8 @@ jobs: with: creds: '{"clientId":"${{ secrets.FAD_CLIENT_ID }}","clientSecret":"${{ secrets.FAD_CLIENT_SECRET }}","subscriptionId":"${{ secrets.FAD_SUBSCRIPTION_ID }}","tenantId":"${{ secrets.FAD_TENANT_ID }}"}' + - name: Fetch release + run: wget https://github.com/${{ github.event.repository.name }}/archive/refs/tags/${{ inputs.tag }}.zip + - name: Deploy Function App - run: az functionapp deployment source config --branch main --manual-integration --name ${{ vars.FAD_FUNCTION_APP_NAME }} --repo-url ${{ github.event.repository.name }} --resource-group ${{ vars.FAD_RESOURCE_GROUP }} + run: az functionapp deployment source config-zip --name ${{ vars.FAD_FUNCTION_APP_NAME }} --resource-group ${{ vars.FAD_RESOURCE_GROUP }} ${{ inputs.tag }}.zip From a86fa2ebf3e7deb824210ba0c454b6740854e431 Mon Sep 17 00:00:00 2001 From: Al West Date: Tue, 9 Jan 2024 17:41:53 +0000 Subject: [PATCH 10/23] Use repository.name and set --src --- .github/workflows/function-app-deploy.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/function-app-deploy.yml b/.github/workflows/function-app-deploy.yml index 7111e6a..9aded47 100644 --- a/.github/workflows/function-app-deploy.yml +++ b/.github/workflows/function-app-deploy.yml @@ -31,7 +31,7 @@ jobs: creds: '{"clientId":"${{ secrets.FAD_CLIENT_ID }}","clientSecret":"${{ secrets.FAD_CLIENT_SECRET }}","subscriptionId":"${{ secrets.FAD_SUBSCRIPTION_ID }}","tenantId":"${{ secrets.FAD_TENANT_ID }}"}' - name: Fetch release - run: wget https://github.com/${{ github.event.repository.name }}/archive/refs/tags/${{ inputs.tag }}.zip + run: wget https://github.com/${{ repository.name }}/archive/refs/tags/${{ inputs.tag }}.zip - name: Deploy Function App - run: az functionapp deployment source config-zip --name ${{ vars.FAD_FUNCTION_APP_NAME }} --resource-group ${{ vars.FAD_RESOURCE_GROUP }} ${{ inputs.tag }}.zip + run: az functionapp deployment source config-zip --name ${{ vars.FAD_FUNCTION_APP_NAME }} --resource-group ${{ vars.FAD_RESOURCE_GROUP }} --src ${{ inputs.tag }}.zip From 5c41b90f592eff46ea00758d569ca1451c9267dc Mon Sep 17 00:00:00 2001 From: Al West Date: Tue, 9 Jan 2024 17:46:11 +0000 Subject: [PATCH 11/23] Update to use github.repository for repo org/name --- .github/workflows/function-app-deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/function-app-deploy.yml b/.github/workflows/function-app-deploy.yml index 9aded47..b53d054 100644 --- a/.github/workflows/function-app-deploy.yml +++ b/.github/workflows/function-app-deploy.yml @@ -31,7 +31,7 @@ jobs: creds: '{"clientId":"${{ secrets.FAD_CLIENT_ID }}","clientSecret":"${{ secrets.FAD_CLIENT_SECRET }}","subscriptionId":"${{ secrets.FAD_SUBSCRIPTION_ID }}","tenantId":"${{ secrets.FAD_TENANT_ID }}"}' - name: Fetch release - run: wget https://github.com/${{ repository.name }}/archive/refs/tags/${{ inputs.tag }}.zip + run: wget https://github.com/${{ github.repository }}/archive/refs/tags/${{ inputs.tag }}.zip - name: Deploy Function App run: az functionapp deployment source config-zip --name ${{ vars.FAD_FUNCTION_APP_NAME }} --resource-group ${{ vars.FAD_RESOURCE_GROUP }} --src ${{ inputs.tag }}.zip From a7e00f4968695fbaadca48b5f45ae3e55220863d Mon Sep 17 00:00:00 2001 From: killij <51908793+killij@users.noreply.github.com> Date: Thu, 11 Jan 2024 08:31:35 +0000 Subject: [PATCH 12/23] feat: Add the estimated reading time to the index (#20) --- src/Childrens-Social-Care-CPD-Indexer/Core/Content.cs | 1 + .../Core/CpdDocument.cs | 9 ++++++--- .../Core/DocumentFetcher.cs | 1 + 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/Childrens-Social-Care-CPD-Indexer/Core/Content.cs b/src/Childrens-Social-Care-CPD-Indexer/Core/Content.cs index 3368689..81a6bc1 100644 --- a/src/Childrens-Social-Care-CPD-Indexer/Core/Content.cs +++ b/src/Childrens-Social-Care-CPD-Indexer/Core/Content.cs @@ -25,6 +25,7 @@ internal class Content : IContent public List? Items { get; set; } public IContent? Navigation { get; set; } public IContent? RelatedContent { get; set; } + public int? EstimatedReadingTime { get; set; } [JsonProperty("$metadata")] public ContentfulMetadata? Metadata { get; set; } diff --git a/src/Childrens-Social-Care-CPD-Indexer/Core/CpdDocument.cs b/src/Childrens-Social-Care-CPD-Indexer/Core/CpdDocument.cs index b1df9da..6ba03cf 100644 --- a/src/Childrens-Social-Care-CPD-Indexer/Core/CpdDocument.cs +++ b/src/Childrens-Social-Care-CPD-Indexer/Core/CpdDocument.cs @@ -20,12 +20,15 @@ internal partial class CpdDocument(string id) [SearchableField(AnalyzerName = LexicalAnalyzerName.Values.EnLucene)] public string? Body { get; set; } - [SimpleField(IsSortable = true, IsFilterable = true)] + [SimpleField(IsSortable = true)] public DateTimeOffset? CreatedAt { get; set; } - [SimpleField(IsSortable = true, IsFilterable = true)] + [SimpleField(IsSortable = true)] public DateTimeOffset? UpdatedAt { get; set; } - + + [SimpleField] + public int? EstimatedReadingTime { get; set; } + [SimpleField(IsFilterable = true, IsFacetable = true)] public IEnumerable? Tags { get; set; } } \ No newline at end of file diff --git a/src/Childrens-Social-Care-CPD-Indexer/Core/DocumentFetcher.cs b/src/Childrens-Social-Care-CPD-Indexer/Core/DocumentFetcher.cs index 95ccd91..9a961f7 100644 --- a/src/Childrens-Social-Care-CPD-Indexer/Core/DocumentFetcher.cs +++ b/src/Childrens-Social-Care-CPD-Indexer/Core/DocumentFetcher.cs @@ -30,6 +30,7 @@ private static CpdDocument BuildResourceDocument(Content content) Body = content.SearchSummary, CreatedAt = content.Sys!.CreatedAt.HasValue ? new DateTimeOffset(content.Sys.CreatedAt.Value) : null, UpdatedAt = content.Sys!.UpdatedAt.HasValue ? new DateTimeOffset(content.Sys.UpdatedAt.Value) : null, + EstimatedReadingTime = content.EstimatedReadingTime, Tags = tags }; } From 1ed33365c9d97d5ba19b343eeeeaa45004dca097 Mon Sep 17 00:00:00 2001 From: killij <51908793+killij@users.noreply.github.com> Date: Mon, 15 Jan 2024 13:59:39 +0000 Subject: [PATCH 13/23] ci: Upload the packaged release to the Github release --- .github/workflows/dotnet-publish.yml | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/.github/workflows/dotnet-publish.yml b/.github/workflows/dotnet-publish.yml index 8967a03..927d758 100644 --- a/.github/workflows/dotnet-publish.yml +++ b/.github/workflows/dotnet-publish.yml @@ -27,11 +27,17 @@ jobs: working-directory: ./src - name: Zip output - run: zip -r ../${{ env.RELEASE_TAG }}.zip ./ + run: zip -r ../${{ env.RELEASE_TAG }}.build.zip ./ working-directory: ./src/publish - - name: Upload artifacts - uses: actions/upload-artifact@v3 + - name: Upload to release + uses: softprops/action-gh-release@v1 + if: startsWith(github.ref, 'refs/tags/') with: - name: resource-indexing-deployment - path: ./src/${{ env.RELEASE_TAG }}.zip \ No newline at end of file + files: ./src/${{ env.RELEASE_TAG }}.build.zip + + # - name: Upload artifacts + # uses: actions/upload-artifact@v3 + # with: + # name: resource-indexing-deployment + # path: ./src/${{ env.RELEASE_TAG }}.zip \ No newline at end of file From 024e848965d8ddc0311bd9849f58965778e334d4 Mon Sep 17 00:00:00 2001 From: killij <51908793+killij@users.noreply.github.com> Date: Mon, 15 Jan 2024 14:20:57 +0000 Subject: [PATCH 14/23] ci: Update function-app-deploy.yml to retrieve the build zip asset from the release (#22) --- .github/workflows/function-app-deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/function-app-deploy.yml b/.github/workflows/function-app-deploy.yml index b53d054..5445e72 100644 --- a/.github/workflows/function-app-deploy.yml +++ b/.github/workflows/function-app-deploy.yml @@ -31,7 +31,7 @@ jobs: creds: '{"clientId":"${{ secrets.FAD_CLIENT_ID }}","clientSecret":"${{ secrets.FAD_CLIENT_SECRET }}","subscriptionId":"${{ secrets.FAD_SUBSCRIPTION_ID }}","tenantId":"${{ secrets.FAD_TENANT_ID }}"}' - name: Fetch release - run: wget https://github.com/${{ github.repository }}/archive/refs/tags/${{ inputs.tag }}.zip + run: wget https://github.com/${{ github.repository }}/releases/download/${{ inputs.tag }}/${{ inputs.tag }}.build.zip - name: Deploy Function App run: az functionapp deployment source config-zip --name ${{ vars.FAD_FUNCTION_APP_NAME }} --resource-group ${{ vars.FAD_RESOURCE_GROUP }} --src ${{ inputs.tag }}.zip From 0f16ec9cb671b317cd485bf72e9cb63d7d27bf22 Mon Sep 17 00:00:00 2001 From: killij <51908793+killij@users.noreply.github.com> Date: Mon, 15 Jan 2024 14:30:20 +0000 Subject: [PATCH 15/23] ci: Fix deployment file name (#23) --- .github/workflows/function-app-deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/function-app-deploy.yml b/.github/workflows/function-app-deploy.yml index 5445e72..6ad4028 100644 --- a/.github/workflows/function-app-deploy.yml +++ b/.github/workflows/function-app-deploy.yml @@ -34,4 +34,4 @@ jobs: run: wget https://github.com/${{ github.repository }}/releases/download/${{ inputs.tag }}/${{ inputs.tag }}.build.zip - name: Deploy Function App - run: az functionapp deployment source config-zip --name ${{ vars.FAD_FUNCTION_APP_NAME }} --resource-group ${{ vars.FAD_RESOURCE_GROUP }} --src ${{ inputs.tag }}.zip + run: az functionapp deployment source config-zip --name ${{ vars.FAD_FUNCTION_APP_NAME }} --resource-group ${{ vars.FAD_RESOURCE_GROUP }} --src ${{ inputs.tag }}.build.zip From aacafe929c3739331db01087e62c89a08b4bdcdb Mon Sep 17 00:00:00 2001 From: killij <51908793+killij@users.noreply.github.com> Date: Tue, 16 Jan 2024 14:31:13 +0000 Subject: [PATCH 16/23] Create build-docker-image.yml --- .github/workflows/build-docker-image.yml | 82 ++++++++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 .github/workflows/build-docker-image.yml diff --git a/.github/workflows/build-docker-image.yml b/.github/workflows/build-docker-image.yml new file mode 100644 index 0000000..0d5c4b6 --- /dev/null +++ b/.github/workflows/build-docker-image.yml @@ -0,0 +1,82 @@ +name: Docker Publish +on: + release: + types: [published] +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + RELEASE_TAG: ${{ github.event.release.tag_name }} +jobs: + build: + name: 'Docker Publish' + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + # This is used to complete the identity challenge with sigstore/fulcio when running outside of PRs. + id-token: write + steps: + # Checkout the release tag version + - name: Checkout repository ${{ env.RELEASE_TAG }} + uses: actions/checkout@v3 + with: + ref: ${{ env.RELEASE_TAG }} + # Get git commit hash + - name: Get short hash + run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_ENV + # Need to lower case the image name for the docker tags when publishing + - name: Downcase IMAGE_NAME variable + run: echo "IMAGE_NAME_LOWER=${IMAGE_NAME,,}" >> $GITHUB_ENV + + # Sort out the image tags + - name: Set initial tag + run: echo "IMAGE_TAGS=${{ env.REGISTRY }}/${{ env.IMAGE_NAME_LOWER }}:${{ env.RELEASE_TAG }}" >> $GITHUB_ENV + - name: Add latest tag if we're not production release + if: contains(env.RELEASE_TAG, 'next') + run: echo "IMAGE_TAGS=${{ env.IMAGE_TAGS }},${{ env.REGISTRY }}/${{ env.IMAGE_NAME_LOWER }}:latest" >> $GITHUB_ENV + #debug + - name: Log the tags + run: echo "Calculated tags value => ${{ env.IMAGE_TAGS }}" + # Setup docker build tool + - name: Setup Docker buildx + uses: docker/setup-buildx-action@v3 + # Login against a Docker registry + - name: Log into registry ${{ env.REGISTRY }} + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + # Extract metadata (tags, labels) for Docker + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + # Build and push Docker image with Buildx (don't push on PR) + - name: Build and push Docker image + id: build-and-push + uses: docker/build-push-action@v5 + with: + context: ./src + file: ./src/Childrens-Social-Care-CPD-Indexer/Dockerfile + push: true + tags: ${{ env.IMAGE_TAGS }} + labels: ${{ steps.meta.outputs.labels }} + build-args: | + VCSREF=${{ env.sha_short }} + VCSTAG=${{ env.RELEASE_TAG }} + cache-from: type=gha + cache-to: type=gha,mode=max + # Sign the resulting Docker image digest. + # This will only write to the public Rekor transparency log when the Docker + # repository is public to avoid leaking data. If you would like to publish + # transparency data even for private images, pass --force to cosign below. + # https://github.com/sigstore/cosign + - name: Install Cosign + uses: sigstore/cosign-installer@v3.2.0 + - name: Check install! + run: cosign version + - name: Sign the published Docker image + # This step uses the identity token to provision an ephemeral certificate against the sigstore community Fulcio instance. + run: echo "${{ steps.meta.outputs.tags }}" | xargs -I {} cosign sign --yes {}@${{ steps.build-and-push.outputs.digest }} \ No newline at end of file From 7a01fb64110677acb8c7208e7c024de09dae048e Mon Sep 17 00:00:00 2001 From: killij <51908793+killij@users.noreply.github.com> Date: Tue, 16 Jan 2024 16:45:54 +0000 Subject: [PATCH 17/23] Convert the function back into a Container App Job --- ...IndexingServiceTests.cs => WorkerTests.cs} | 31 ++++-- src/Childrens-Social-Care-CPD-Indexer.sln | 24 ++--- .../Childrens-Social-Care-CPD-Indexer.csproj | 39 +++---- .../Dockerfile | 11 +- .../Indexer.cs | 37 ------- .../Program.cs | 101 +++++++++--------- .../Properties/launchSettings.json | 13 +-- .../Properties/serviceDependencies.json | 11 -- .../Properties/serviceDependencies.local.json | 11 -- .../Worker.cs | 45 ++++++++ .../appsettings.Development.json | 8 ++ .../appsettings.json | 8 ++ .../host.json | 12 --- 13 files changed, 167 insertions(+), 184 deletions(-) rename src/Childrens-Social-Care-CPD-Indexer.Tests/{IndexingServiceTests.cs => WorkerTests.cs} (66%) delete mode 100644 src/Childrens-Social-Care-CPD-Indexer/Indexer.cs delete mode 100644 src/Childrens-Social-Care-CPD-Indexer/Properties/serviceDependencies.json delete mode 100644 src/Childrens-Social-Care-CPD-Indexer/Properties/serviceDependencies.local.json create mode 100644 src/Childrens-Social-Care-CPD-Indexer/Worker.cs create mode 100644 src/Childrens-Social-Care-CPD-Indexer/appsettings.Development.json create mode 100644 src/Childrens-Social-Care-CPD-Indexer/appsettings.json delete mode 100644 src/Childrens-Social-Care-CPD-Indexer/host.json diff --git a/src/Childrens-Social-Care-CPD-Indexer.Tests/IndexingServiceTests.cs b/src/Childrens-Social-Care-CPD-Indexer.Tests/WorkerTests.cs similarity index 66% rename from src/Childrens-Social-Care-CPD-Indexer.Tests/IndexingServiceTests.cs rename to src/Childrens-Social-Care-CPD-Indexer.Tests/WorkerTests.cs index 782e78a..3b00d8b 100644 --- a/src/Childrens-Social-Care-CPD-Indexer.Tests/IndexingServiceTests.cs +++ b/src/Childrens-Social-Care-CPD-Indexer.Tests/WorkerTests.cs @@ -1,25 +1,33 @@ using Childrens_Social_Care_CPD_Indexer.Core; -using Microsoft.ApplicationInsights; -using Microsoft.Azure.Functions.Worker; +using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using NSubstitute.ExceptionExtensions; namespace Childrens_Social_Care_CPD_Indexer.Tests; -public class IndexingServiceTests +public class WorkerTests { - private ILogger _logger; + private ILogger _logger; private IResourcesIndexerConfig _config; private IResourcesIndexer _indexer; - private Indexer _sut; + private IHostApplicationLifetime _hostingApplicationLifetime; + private Worker _sut; [SetUp] public void Setup() { - _logger = Substitute.For>(); + _logger = Substitute.For>(); _config = Substitute.For(); _indexer = Substitute.For(); - _sut = new Indexer(_logger, _indexer, _config); + _hostingApplicationLifetime = Substitute.For(); + + _sut = new Worker(_logger, _indexer, _config, _hostingApplicationLifetime); + } + + [TearDown] + public void Teardown() + { + _sut.Dispose(); } [Test] @@ -27,9 +35,10 @@ public async Task StartAsync_Deletes_Index_If_Configured() { // arrange _config.RecreateIndex.Returns(true); + var cancellationTokenSource = new CancellationTokenSource(); // act - await _sut.Run(new TimerInfo()); + await _sut.StartAsync(cancellationTokenSource.Token); // assert await _indexer.Received(1).DeleteIndexAsync(Arg.Any(), Arg.Any()); @@ -41,9 +50,10 @@ public async Task StartAsync_Populates_Index() { // arrange _config.RecreateIndex.Returns(false); + var cancellationTokenSource = new CancellationTokenSource(); // act - await _sut.Run(new TimerInfo()); + await _sut.StartAsync(cancellationTokenSource.Token); // assert await _indexer.Received(1).PopulateIndexAsync(Arg.Any(), Arg.Any(), Arg.Any()); @@ -56,9 +66,10 @@ public async Task StartAsync_Logs_Exception() var exception = new InvalidOperationException(); _config.RecreateIndex.Returns(true); _indexer.DeleteIndexAsync(Arg.Any(), Arg.Any()).Throws(exception); + var cancellationTokenSource = new CancellationTokenSource(); // act - await _sut.Run(new TimerInfo()); + await _sut.StartAsync(cancellationTokenSource.Token); // assert _logger.Received(1).LogError(exception, "Unhandled exception occured"); diff --git a/src/Childrens-Social-Care-CPD-Indexer.sln b/src/Childrens-Social-Care-CPD-Indexer.sln index 21908a1..0a76fb1 100644 --- a/src/Childrens-Social-Care-CPD-Indexer.sln +++ b/src/Childrens-Social-Care-CPD-Indexer.sln @@ -1,11 +1,11 @@  Microsoft Visual Studio Solution File, Format Version 12.00 # Visual Studio Version 17 -VisualStudioVersion = 17.8.34322.80 +VisualStudioVersion = 17.8.34330.188 MinimumVisualStudioVersion = 10.0.40219.1 -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Childrens-Social-Care-CPD-Indexer.Tests", "Childrens-Social-Care-CPD-Indexer.Tests\Childrens-Social-Care-CPD-Indexer.Tests.csproj", "{6984BF56-808E-4294-949D-FFE4B02CCE16}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Childrens-Social-Care-CPD-Indexer", "Childrens-Social-Care-CPD-Indexer\Childrens-Social-Care-CPD-Indexer.csproj", "{F2710DEC-7B59-484A-9D04-F73B1E09563A}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Childrens-Social-Care-CPD-Indexer", "Childrens-Social-Care-CPD-Indexer\Childrens-Social-Care-CPD-Indexer.csproj", "{3555944D-1913-4979-B5BF-991C7064613E}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Childrens-Social-Care-CPD-Indexer.Tests", "Childrens-Social-Care-CPD-Indexer.Tests\Childrens-Social-Care-CPD-Indexer.Tests.csproj", "{F0344BA1-9B92-42A7-B650-B8064EDD2B9E}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -13,19 +13,19 @@ Global Release|Any CPU = Release|Any CPU EndGlobalSection GlobalSection(ProjectConfigurationPlatforms) = postSolution - {6984BF56-808E-4294-949D-FFE4B02CCE16}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {6984BF56-808E-4294-949D-FFE4B02CCE16}.Debug|Any CPU.Build.0 = Debug|Any CPU - {6984BF56-808E-4294-949D-FFE4B02CCE16}.Release|Any CPU.ActiveCfg = Release|Any CPU - {6984BF56-808E-4294-949D-FFE4B02CCE16}.Release|Any CPU.Build.0 = Release|Any CPU - {3555944D-1913-4979-B5BF-991C7064613E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {3555944D-1913-4979-B5BF-991C7064613E}.Debug|Any CPU.Build.0 = Debug|Any CPU - {3555944D-1913-4979-B5BF-991C7064613E}.Release|Any CPU.ActiveCfg = Release|Any CPU - {3555944D-1913-4979-B5BF-991C7064613E}.Release|Any CPU.Build.0 = Release|Any CPU + {F2710DEC-7B59-484A-9D04-F73B1E09563A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {F2710DEC-7B59-484A-9D04-F73B1E09563A}.Debug|Any CPU.Build.0 = Debug|Any CPU + {F2710DEC-7B59-484A-9D04-F73B1E09563A}.Release|Any CPU.ActiveCfg = Release|Any CPU + {F2710DEC-7B59-484A-9D04-F73B1E09563A}.Release|Any CPU.Build.0 = Release|Any CPU + {F0344BA1-9B92-42A7-B650-B8064EDD2B9E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {F0344BA1-9B92-42A7-B650-B8064EDD2B9E}.Debug|Any CPU.Build.0 = Debug|Any CPU + {F0344BA1-9B92-42A7-B650-B8064EDD2B9E}.Release|Any CPU.ActiveCfg = Release|Any CPU + {F0344BA1-9B92-42A7-B650-B8064EDD2B9E}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution - SolutionGuid = {EEC0CA75-7434-4AC0-9F45-31CA6CC7D59A} + SolutionGuid = {5F3E3B0D-C6EA-4304-851B-75680DAB332E} EndGlobalSection EndGlobal diff --git a/src/Childrens-Social-Care-CPD-Indexer/Childrens-Social-Care-CPD-Indexer.csproj b/src/Childrens-Social-Care-CPD-Indexer/Childrens-Social-Care-CPD-Indexer.csproj index 1692839..5c81250 100644 --- a/src/Childrens-Social-Care-CPD-Indexer/Childrens-Social-Care-CPD-Indexer.csproj +++ b/src/Childrens-Social-Care-CPD-Indexer/Childrens-Social-Care-CPD-Indexer.csproj @@ -1,39 +1,24 @@ - + + net8.0 - v4 - Exe - enable enable + enable + dotnet-Childrens_Social_Care_CPD_Indexer-907bada8-7f54-479d-b04e-45ec4f148332 Childrens_Social_Care_CPD_Indexer - /home/site/wwwroot Linux - - - - + + + + + - - - - - + + - - - PreserveNewest - - - PreserveNewest - Never - - - - - - \ No newline at end of file + diff --git a/src/Childrens-Social-Care-CPD-Indexer/Dockerfile b/src/Childrens-Social-Care-CPD-Indexer/Dockerfile index c4a2687..9b63d70 100644 --- a/src/Childrens-Social-Care-CPD-Indexer/Dockerfile +++ b/src/Childrens-Social-Care-CPD-Indexer/Dockerfile @@ -1,8 +1,8 @@ #See https://aka.ms/customizecontainer to learn how to customize your debug container and how Visual Studio uses this Dockerfile to build your images for faster debugging. -FROM mcr.microsoft.com/azure-functions/dotnet-isolated:4-dotnet-isolated8.0 AS base -WORKDIR /home/site/wwwroot -EXPOSE 8080 +FROM mcr.microsoft.com/dotnet/runtime:8.0 AS base +USER app +WORKDIR /app FROM mcr.microsoft.com/dotnet/sdk:8.0 AS build ARG BUILD_CONFIGURATION=Release @@ -18,7 +18,6 @@ ARG BUILD_CONFIGURATION=Release RUN dotnet publish "./Childrens-Social-Care-CPD-Indexer.csproj" -c $BUILD_CONFIGURATION -o /app/publish /p:UseAppHost=false FROM base AS final -WORKDIR /home/site/wwwroot +WORKDIR /app COPY --from=publish /app/publish . -ENV AzureWebJobsScriptRoot=/home/site/wwwroot \ - AzureFunctionsJobHost__Logging__Console__IsEnabled=true \ No newline at end of file +ENTRYPOINT ["dotnet", "Childrens-Social-Care-CPD-Indexer.dll"] \ No newline at end of file diff --git a/src/Childrens-Social-Care-CPD-Indexer/Indexer.cs b/src/Childrens-Social-Care-CPD-Indexer/Indexer.cs deleted file mode 100644 index b7c069d..0000000 --- a/src/Childrens-Social-Care-CPD-Indexer/Indexer.cs +++ /dev/null @@ -1,37 +0,0 @@ -using Childrens_Social_Care_CPD_Indexer.Core; -using Microsoft.Azure.Functions.Worker; -using Microsoft.Extensions.Logging; - -namespace Childrens_Social_Care_CPD_Indexer; - -public class Indexer(ILogger logger, IResourcesIndexer resourcesIndexer, IResourcesIndexerConfig config) -{ - [Function("IndexResources")] - public async Task Run([TimerTrigger("0 0 * * SUN" - #if DEBUG - , RunOnStartup= true - #endif - )] TimerInfo myTimer, CancellationToken cancellationToken = default) - { - logger.LogInformation("Indexing started at: {startTime}", DateTime.Now); - try - { - if (config.RecreateIndex) - { - await resourcesIndexer.DeleteIndexAsync(config.IndexName, cancellationToken); - } - - await resourcesIndexer.CreateIndexAsync(config.IndexName, cancellationToken); - await resourcesIndexer.PopulateIndexAsync(config.IndexName, config.BatchSize, cancellationToken); - - } - catch (Exception ex) - { - logger.LogError(ex, "Unhandled exception occured"); - } - finally - { - logger.LogInformation("Indexing finished at: {finishTime}", DateTime.Now); - } - } -} diff --git a/src/Childrens-Social-Care-CPD-Indexer/Program.cs b/src/Childrens-Social-Care-CPD-Indexer/Program.cs index f3352b8..e61132b 100644 --- a/src/Childrens-Social-Care-CPD-Indexer/Program.cs +++ b/src/Childrens-Social-Care-CPD-Indexer/Program.cs @@ -1,59 +1,56 @@ -using Childrens_Social_Care_CPD_Indexer.Core; -using Childrens_Social_Care_CPD_Indexer; -using Microsoft.Azure.Functions.Worker; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Hosting; -using Microsoft.ApplicationInsights.WorkerService; using Azure.Search.Documents.Indexes; using Azure; +using Childrens_Social_Care_CPD_Indexer; +using Childrens_Social_Care_CPD_Indexer.Core; using Contentful.Core.Configuration; using Contentful.Core; +using Microsoft.ApplicationInsights.WorkerService; using Microsoft.ApplicationInsights; -using Microsoft.Extensions.DependencyInjection.Extensions; -using Microsoft.Extensions.Logging; +using System.Diagnostics.CodeAnalysis; + +var builder = Host.CreateApplicationBuilder(args); -var host = new HostBuilder() - .ConfigureFunctionsWorkerDefaults() - .ConfigureServices((context, services) => +builder.Services.AddTransient(); +var config = new ResourcesIndexerConfig(builder.Configuration); + +// Logging +var options = new ApplicationInsightsServiceOptions() +{ + ApplicationVersion = config.ApplicationVersion, + ConnectionString = config.AppInsightsConnectionString, +}; + +builder.Services.AddApplicationInsightsTelemetryWorkerService(options); + +// Code dependencies +builder.Services.AddTransient(); +builder.Services.AddTransient(servicesProvider => { + var httpClient = servicesProvider.GetRequiredService(); + var resourcesIndexerConfig = servicesProvider.GetRequiredService(); + var contentfulOptions = new ContentfulOptions() { - services.AddTransient(); - var config = new ResourcesIndexerConfig(context.Configuration); - - - // Logging - var options = new ApplicationInsightsServiceOptions() - { - ApplicationVersion = config.ApplicationVersion, - ConnectionString = config.AppInsightsConnectionString, - }; - - services.AddApplicationInsightsTelemetryWorkerService(options); - services.ConfigureFunctionsApplicationInsights(); - - // Code dependencies - services.TryAddTransient(); - services.AddTransient(servicesProvider => { - var httpClient = servicesProvider.GetRequiredService(); - var resourcesIndexerConfig = servicesProvider.GetRequiredService(); - var contentfulOptions = new ContentfulOptions() - { - DeliveryApiKey = resourcesIndexerConfig.ContentfulApiKey, - SpaceId = resourcesIndexerConfig.ContentfulSpaceId, - Environment = resourcesIndexerConfig.ContentfulEnvironmentId - }; - return new ContentfulClient(httpClient, contentfulOptions); - }); - services.AddTransient(); - services.AddTransient(servicesProvider => { - var logger = servicesProvider.GetRequiredService>(); - var config = servicesProvider.GetRequiredService(); - var documentFetcher = servicesProvider.GetRequiredService(); - var searchEndpointUri = new Uri(config.Endpoint); - var searchIndexClient = new SearchIndexClient(searchEndpointUri, new AzureKeyCredential(config.ApiKey)); - var telemtryClient = servicesProvider.GetRequiredService(); - return new ResourcesIndexer(searchIndexClient, documentFetcher, logger, telemtryClient); - }); - }) - .Build(); - -host.Run(); + DeliveryApiKey = resourcesIndexerConfig.ContentfulApiKey, + SpaceId = resourcesIndexerConfig.ContentfulSpaceId, + Environment = resourcesIndexerConfig.ContentfulEnvironmentId + }; + return new ContentfulClient(httpClient, contentfulOptions); +}); + +builder.Services.AddTransient(); +builder.Services.AddTransient(servicesProvider => { + var logger = servicesProvider.GetRequiredService>(); + var config = servicesProvider.GetRequiredService(); + var documentFetcher = servicesProvider.GetRequiredService(); + var searchEndpointUri = new Uri(config.Endpoint); + var searchIndexClient = new SearchIndexClient(searchEndpointUri, new AzureKeyCredential(config.ApiKey)); + var telemtryClient = servicesProvider.GetRequiredService(); + return new ResourcesIndexer(searchIndexClient, documentFetcher, logger, telemtryClient); +}); + +builder.Services.AddHostedService(); + +var host = builder.Build(); +await host.RunAsync(); + +[ExcludeFromCodeCoverage] +public partial class Program() { } \ No newline at end of file diff --git a/src/Childrens-Social-Care-CPD-Indexer/Properties/launchSettings.json b/src/Childrens-Social-Care-CPD-Indexer/Properties/launchSettings.json index 3939375..cae296e 100644 --- a/src/Childrens-Social-Care-CPD-Indexer/Properties/launchSettings.json +++ b/src/Childrens-Social-Care-CPD-Indexer/Properties/launchSettings.json @@ -2,13 +2,14 @@ "profiles": { "Childrens_Social_Care_CPD_Indexer": { "commandName": "Project", - "commandLineArgs": "--port 7016" + "environmentVariables": { + "DOTNET_ENVIRONMENT": "Development" + }, + "dotnetRunMessages": true }, "Docker": { - "commandName": "Docker", - "launchUrl": "{Scheme}://{ServiceHost}:{ServicePort}", - "httpPort": 31453, - "useSSL": false + "commandName": "Docker" } - } + }, + "$schema": "http://json.schemastore.org/launchsettings.json" } \ No newline at end of file diff --git a/src/Childrens-Social-Care-CPD-Indexer/Properties/serviceDependencies.json b/src/Childrens-Social-Care-CPD-Indexer/Properties/serviceDependencies.json deleted file mode 100644 index df4dcc9..0000000 --- a/src/Childrens-Social-Care-CPD-Indexer/Properties/serviceDependencies.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "dependencies": { - "appInsights1": { - "type": "appInsights" - }, - "storage1": { - "type": "storage", - "connectionId": "AzureWebJobsStorage" - } - } -} \ No newline at end of file diff --git a/src/Childrens-Social-Care-CPD-Indexer/Properties/serviceDependencies.local.json b/src/Childrens-Social-Care-CPD-Indexer/Properties/serviceDependencies.local.json deleted file mode 100644 index b804a28..0000000 --- a/src/Childrens-Social-Care-CPD-Indexer/Properties/serviceDependencies.local.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "dependencies": { - "appInsights1": { - "type": "appInsights.sdk" - }, - "storage1": { - "type": "storage.emulator", - "connectionId": "AzureWebJobsStorage" - } - } -} \ No newline at end of file diff --git a/src/Childrens-Social-Care-CPD-Indexer/Worker.cs b/src/Childrens-Social-Care-CPD-Indexer/Worker.cs new file mode 100644 index 0000000..9db030e --- /dev/null +++ b/src/Childrens-Social-Care-CPD-Indexer/Worker.cs @@ -0,0 +1,45 @@ +using Childrens_Social_Care_CPD_Indexer.Core; + +namespace Childrens_Social_Care_CPD_Indexer; + +public class Worker : BackgroundService +{ + private readonly ILogger _logger; + private readonly IResourcesIndexer _resourcesIndexer; + private readonly IResourcesIndexerConfig _config; + private readonly IHostApplicationLifetime _hostApplicationLifetime; + + public Worker(ILogger logger, IResourcesIndexer resourcesIndexer, IResourcesIndexerConfig config, IHostApplicationLifetime hostApplicationLifetime) + { + _logger = logger; + _resourcesIndexer = resourcesIndexer; + _config = config; + _hostApplicationLifetime = hostApplicationLifetime; + } + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) => + await DoWork(stoppingToken).ContinueWith(task => _hostApplicationLifetime.StopApplication(), stoppingToken); + + private async Task DoWork(CancellationToken stoppingToken) + { + _logger.LogInformation("Indexing started at: {startTime}", DateTime.Now); + try + { + if (_config.RecreateIndex) + { + await _resourcesIndexer.DeleteIndexAsync(_config.IndexName, stoppingToken); + } + await _resourcesIndexer.CreateIndexAsync(_config.IndexName, stoppingToken); + await _resourcesIndexer.PopulateIndexAsync(_config.IndexName, _config.BatchSize, stoppingToken); + + } + catch (Exception ex) + { + _logger.LogError(ex, "Unhandled exception occured"); + } + finally + { + _logger.LogInformation("Indexing finished at: {finishTime}", DateTime.Now); + } + } +} diff --git a/src/Childrens-Social-Care-CPD-Indexer/appsettings.Development.json b/src/Childrens-Social-Care-CPD-Indexer/appsettings.Development.json new file mode 100644 index 0000000..b2dcdb6 --- /dev/null +++ b/src/Childrens-Social-Care-CPD-Indexer/appsettings.Development.json @@ -0,0 +1,8 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.Hosting.Lifetime": "Information" + } + } +} diff --git a/src/Childrens-Social-Care-CPD-Indexer/appsettings.json b/src/Childrens-Social-Care-CPD-Indexer/appsettings.json new file mode 100644 index 0000000..b2dcdb6 --- /dev/null +++ b/src/Childrens-Social-Care-CPD-Indexer/appsettings.json @@ -0,0 +1,8 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.Hosting.Lifetime": "Information" + } + } +} diff --git a/src/Childrens-Social-Care-CPD-Indexer/host.json b/src/Childrens-Social-Care-CPD-Indexer/host.json deleted file mode 100644 index ee5cf5f..0000000 --- a/src/Childrens-Social-Care-CPD-Indexer/host.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "version": "2.0", - "logging": { - "applicationInsights": { - "samplingSettings": { - "isEnabled": true, - "excludedTypes": "Request" - }, - "enableLiveMetricsFilters": true - } - } -} \ No newline at end of file From 808e82e8f5ce671566c6b66a0f6a9d78ce443d95 Mon Sep 17 00:00:00 2001 From: killij <51908793+killij@users.noreply.github.com> Date: Tue, 16 Jan 2024 16:48:20 +0000 Subject: [PATCH 18/23] Delete unrequired github actions --- .github/workflows/dotnet-publish.yml | 43 ----------------------- .github/workflows/function-app-deploy.yml | 37 ------------------- 2 files changed, 80 deletions(-) delete mode 100644 .github/workflows/dotnet-publish.yml delete mode 100644 .github/workflows/function-app-deploy.yml diff --git a/.github/workflows/dotnet-publish.yml b/.github/workflows/dotnet-publish.yml deleted file mode 100644 index 927d758..0000000 --- a/.github/workflows/dotnet-publish.yml +++ /dev/null @@ -1,43 +0,0 @@ -name: Dotnet Publish - -on: - release: - types: [published] - -env: - RELEASE_TAG: ${{ github.event.release.tag_name }} - -jobs: - build: - name: 'Build Deployment Package' - runs-on: ubuntu-latest - steps: - - name: Checkout repository ${{ env.RELEASE_TAG }} - uses: actions/checkout@v3 - with: - ref: ${{ env.RELEASE_TAG }} - - - name: Setup dotnet - uses: actions/setup-dotnet@v3 - with: - dotnet-version: '8.x' - - - name: dotnet publish - run: dotnet publish ./Childrens-Social-Care-CPD-Indexer/Childrens-Social-Care-CPD-Indexer.csproj -c Release -o ./publish - working-directory: ./src - - - name: Zip output - run: zip -r ../${{ env.RELEASE_TAG }}.build.zip ./ - working-directory: ./src/publish - - - name: Upload to release - uses: softprops/action-gh-release@v1 - if: startsWith(github.ref, 'refs/tags/') - with: - files: ./src/${{ env.RELEASE_TAG }}.build.zip - - # - name: Upload artifacts - # uses: actions/upload-artifact@v3 - # with: - # name: resource-indexing-deployment - # path: ./src/${{ env.RELEASE_TAG }}.zip \ No newline at end of file diff --git a/.github/workflows/function-app-deploy.yml b/.github/workflows/function-app-deploy.yml deleted file mode 100644 index 6ad4028..0000000 --- a/.github/workflows/function-app-deploy.yml +++ /dev/null @@ -1,37 +0,0 @@ -name: Function App Deploy - -on: - workflow_dispatch: - inputs: - tag: - description: 'App Tag to deploy' - required: true - default: 'v1.0.0' - workspace: - description: 'Environment to deploy to' - required: true - default: 'Dev' - type: choice - options: - - Dev - - Test - - Pre-Prod - - Prod - - Load-Test - -jobs: - build: - name: 'Function App Deploy' - runs-on: ubuntu-latest - environment: ${{ inputs.workspace }} - steps: - - name: Sign in to Azure - uses: azure/login@v1 - with: - creds: '{"clientId":"${{ secrets.FAD_CLIENT_ID }}","clientSecret":"${{ secrets.FAD_CLIENT_SECRET }}","subscriptionId":"${{ secrets.FAD_SUBSCRIPTION_ID }}","tenantId":"${{ secrets.FAD_TENANT_ID }}"}' - - - name: Fetch release - run: wget https://github.com/${{ github.repository }}/releases/download/${{ inputs.tag }}/${{ inputs.tag }}.build.zip - - - name: Deploy Function App - run: az functionapp deployment source config-zip --name ${{ vars.FAD_FUNCTION_APP_NAME }} --resource-group ${{ vars.FAD_RESOURCE_GROUP }} --src ${{ inputs.tag }}.build.zip From 901520bde1efdfa188ded91df4fd628f506eb439 Mon Sep 17 00:00:00 2001 From: killij <51908793+killij@users.noreply.github.com> Date: Tue, 16 Jan 2024 17:24:31 +0000 Subject: [PATCH 19/23] Make sure *everything* gets logged to ApplicationInsights --- src/Childrens-Social-Care-CPD-Indexer/appsettings.json | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/Childrens-Social-Care-CPD-Indexer/appsettings.json b/src/Childrens-Social-Care-CPD-Indexer/appsettings.json index b2dcdb6..96a7301 100644 --- a/src/Childrens-Social-Care-CPD-Indexer/appsettings.json +++ b/src/Childrens-Social-Care-CPD-Indexer/appsettings.json @@ -3,6 +3,13 @@ "LogLevel": { "Default": "Information", "Microsoft.Hosting.Lifetime": "Information" + }, + "ApplicationInsights": { + "LogLevel": { + "Default": "Information", + "Microsoft": "Information", + "Microsoft.Hosting.Lifetime": "Information" + } } } } From a6c963978f611e43f9ca9c0e1c8c1a9d1b4ac55e Mon Sep 17 00:00:00 2001 From: killij <51908793+killij@users.noreply.github.com> Date: Fri, 19 Jan 2024 13:20:00 +0000 Subject: [PATCH 20/23] feat: Add Azure Key Vault as a configuration provider and compact the config into a section (#25) --- README.md | 52 ++++++++++----- .../ResourcesIndexerConfigTests.cs | 58 ----------------- .../WorkerTests.cs | 10 +-- .../ApplicationConfiguration.cs | 65 +++++++++++++++++++ .../Childrens-Social-Care-CPD-Indexer.csproj | 4 +- .../Core/IResourcesIndexerConfig.cs | 15 ----- .../Program.cs | 32 +++++---- .../ResourcesIndexerConfig.cs | 18 ----- .../Worker.cs | 14 ++-- 9 files changed, 138 insertions(+), 130 deletions(-) delete mode 100644 src/Childrens-Social-Care-CPD-Indexer.Tests/ResourcesIndexerConfigTests.cs create mode 100644 src/Childrens-Social-Care-CPD-Indexer/ApplicationConfiguration.cs delete mode 100644 src/Childrens-Social-Care-CPD-Indexer/Core/IResourcesIndexerConfig.cs delete mode 100644 src/Childrens-Social-Care-CPD-Indexer/ResourcesIndexerConfig.cs diff --git a/README.md b/README.md index bdfe008..1ba1f3c 100644 --- a/README.md +++ b/README.md @@ -15,19 +15,41 @@ dotnet build dotnet test ``` -## Environment variables -The following environment variables will need to be specified for the application to run: - -| Variable name | Type/Value | Description | -| ------------- | ------------- | ------------- | -| CPD_SEARCH_API_KEY | string | The Azure AI Search API key | -| CPD_INSTRUMENTATION_CONNECTIONSTRING | string | The Azure ApplicationInsights connection string | -| VCS-TAG | string | The application version | -| CPD_SEARCH_BATCH_SIZE | integer (e.g. 10/20 etc) | The batch size for queries into Contentful | -| CPD_SEARCH_ENDPOINT | string | The Azure AI Search endpoint | -| CPD_SEARCH_INDEX_NAME | string | The Azure AI Search index name to access/create | -| CPD_DELIVERY_KEY | string | The Contentful delivery API key | -| CPD_CONTENTFUL_ENVIRONMENT | string | The Contentful enviroment id | -| CPD_SPACE_ID | string | The Contentful space id | -| CPD_SEARCH_RECREATE_INDEX_ON_REBUILD | boolean (true/false) | Whether to delete the index and recreate before populating | +## Configuration +Three configuration values are required to be set in the environment: +* ``CPD_KEY_VAULT_NAME`` - the name of the keyvault to retrieve configuration from in a deployed environment +* ``CPD_CONFIG_SECTION_NAME`` - this is the key name of the root of the configuration section that stores most of the application config +* ``__Application__Version`` - the application version, note the prefix should be set to the above value, for example if `CPD_CONFIG_SECTION_NAME` was set to `DEV` then it would be `DEV__Application__Version` +The remaining application configuration is stored within an IConfiguration section, named using (1) above. In a deployed environment this should be stored in Azure Key Vault. + +The configuration under the root key takes the following hierarchical structure: +``` + ApplicationInsights + ConnectionString + Contentful + DeliveryKey + Environment + SpaceId + SearchIndexing + ApiKey + BatchSize + Endpoint + IndexName + RecreateIndex +``` + + +### Local developer configuration +For development, this structure can be configured in the `secrets.json` file for the project. +Also make sure you set `LOCAL_ENVIRONMENT` to true, otherwise the application will try to initialise key vault. `LOCAL_ENVIRONMENT` is not required for any other scenario. + +### Deployed environments +Key names for secrets are built from the full path to the config key, separating each level with two dash characters (--). So if `CPD_CONFIG_SECTION_NAME` is set to `DEV` then it would be: +``` +DEV--ApplicationInsights--ConnectionString +DEV--ApplicationVersion +DEV--Contentful--DeliveryKey +... +``` +Note this format is for **Key Vault only**. \ No newline at end of file diff --git a/src/Childrens-Social-Care-CPD-Indexer.Tests/ResourcesIndexerConfigTests.cs b/src/Childrens-Social-Care-CPD-Indexer.Tests/ResourcesIndexerConfigTests.cs deleted file mode 100644 index 2790358..0000000 --- a/src/Childrens-Social-Care-CPD-Indexer.Tests/ResourcesIndexerConfigTests.cs +++ /dev/null @@ -1,58 +0,0 @@ -using Microsoft.Extensions.Configuration; - -namespace Childrens_Social_Care_CPD_Indexer.Tests; - -public class ResourcesIndexerConfigTests -{ - [TestCase("ApiKey", "CPD_SEARCH_API_KEY", "Api key", "Api key")] - [TestCase("AppInsightsConnectionString", "CPD_INSTRUMENTATION_CONNECTIONSTRING", "Connection string", "Connection string")] - [TestCase("ApplicationVersion", "VCS-TAG", "1.0.0", "1.0.0")] - [TestCase("BatchSize", "CPD_SEARCH_BATCH_SIZE", "2", 2)] - [TestCase("Endpoint", "CPD_SEARCH_ENDPOINT", "Endpoint", "Endpoint")] - [TestCase("IndexName", "CPD_SEARCH_INDEX_NAME", "Index name", "Index name")] - [TestCase("ContentfulApiKey", "CPD_DELIVERY_KEY", "Contentful api key", "Contentful api key")] - [TestCase("ContentfulEnvironmentId", "CPD_CONTENTFUL_ENVIRONMENT", "Environment id", "Environment id")] - [TestCase("ContentfulSpaceId", "CPD_SPACE_ID", "Space id", "Space id")] - [TestCase("RecreateIndex", "CPD_SEARCH_RECREATE_INDEX_ON_REBUILD", "true", true)] - [TestCase("RecreateIndex", "CPD_SEARCH_RECREATE_INDEX_ON_REBUILD", "false", false)] - public void Config_Returns_Values(string propName, string key, string value, object expected) - { - // arrange - var inMemorySettings = new Dictionary { - {key, value}, - }; - var configuration = new ConfigurationBuilder().AddInMemoryCollection(inMemorySettings!).Build(); - var sut = new ResourcesIndexerConfig(configuration); - var propertyInfo = typeof(ResourcesIndexerConfig).Properties().Single(x => x.Name == propName); - - // act - var actual = propertyInfo.GetValue(sut); - - // assert - actual.Should().Be(expected); - } - - [TestCase("ApiKey", "")] - [TestCase("AppInsightsConnectionString", "")] - [TestCase("ApplicationVersion", "")] - [TestCase("BatchSize", 20)] - [TestCase("Endpoint", "")] - [TestCase("IndexName", "")] - [TestCase("ContentfulApiKey", "")] - [TestCase("ContentfulEnvironmentId", "")] - [TestCase("ContentfulSpaceId", "")] - [TestCase("RecreateIndex", true)] - public void Config_Returns_Default_Values(string propName, object expected) - { - // arrange - var configuration = new ConfigurationBuilder().AddInMemoryCollection(new Dictionary()!).Build(); - var sut = new ResourcesIndexerConfig(configuration); - var propertyInfo = typeof(ResourcesIndexerConfig).Properties().Single(x => x.Name == propName); - - // act - var actual = propertyInfo.GetValue(sut); - - // assert - actual.Should().Be(expected); - } -} \ No newline at end of file diff --git a/src/Childrens-Social-Care-CPD-Indexer.Tests/WorkerTests.cs b/src/Childrens-Social-Care-CPD-Indexer.Tests/WorkerTests.cs index 3b00d8b..1a663c6 100644 --- a/src/Childrens-Social-Care-CPD-Indexer.Tests/WorkerTests.cs +++ b/src/Childrens-Social-Care-CPD-Indexer.Tests/WorkerTests.cs @@ -8,7 +8,7 @@ namespace Childrens_Social_Care_CPD_Indexer.Tests; public class WorkerTests { private ILogger _logger; - private IResourcesIndexerConfig _config; + private IApplicationConfiguration _config; private IResourcesIndexer _indexer; private IHostApplicationLifetime _hostingApplicationLifetime; private Worker _sut; @@ -17,7 +17,7 @@ public class WorkerTests public void Setup() { _logger = Substitute.For>(); - _config = Substitute.For(); + _config = Substitute.For(); _indexer = Substitute.For(); _hostingApplicationLifetime = Substitute.For(); @@ -34,7 +34,7 @@ public void Teardown() public async Task StartAsync_Deletes_Index_If_Configured() { // arrange - _config.RecreateIndex.Returns(true); + _config.SearchIndexing.RecreateIndex.Returns(true); var cancellationTokenSource = new CancellationTokenSource(); // act @@ -49,7 +49,7 @@ public async Task StartAsync_Deletes_Index_If_Configured() public async Task StartAsync_Populates_Index() { // arrange - _config.RecreateIndex.Returns(false); + _config.SearchIndexing.RecreateIndex.Returns(false); var cancellationTokenSource = new CancellationTokenSource(); // act @@ -64,7 +64,7 @@ public async Task StartAsync_Logs_Exception() { // arrange var exception = new InvalidOperationException(); - _config.RecreateIndex.Returns(true); + _config.SearchIndexing.RecreateIndex.Returns(true); _indexer.DeleteIndexAsync(Arg.Any(), Arg.Any()).Throws(exception); var cancellationTokenSource = new CancellationTokenSource(); diff --git a/src/Childrens-Social-Care-CPD-Indexer/ApplicationConfiguration.cs b/src/Childrens-Social-Care-CPD-Indexer/ApplicationConfiguration.cs new file mode 100644 index 0000000..11aa35a --- /dev/null +++ b/src/Childrens-Social-Care-CPD-Indexer/ApplicationConfiguration.cs @@ -0,0 +1,65 @@ +using System.Diagnostics.CodeAnalysis; + +namespace Childrens_Social_Care_CPD_Indexer; + +public interface IApplicationInsightsConfig +{ + string ConnectionString { get; set; } +} + +public interface IContentfulConfig +{ + string DeliveryKey { get; set; } + string Environment { get; set; } + string SpaceId { get; set; } +} + +public interface ISearchIndexingConfig +{ + string ApiKey { get; set; } + int BatchSize { get; set; } + string Endpoint { get; set; } + string IndexName { get; set; } + bool RecreateIndex { get; set; } +} + +public interface IApplicationConfiguration +{ + string ApplicationVersion { get; set; } + IApplicationInsightsConfig ApplicationInsights { get; set; } + IContentfulConfig Contentful { get; set; } + ISearchIndexingConfig SearchIndexing { get; set; } +} + +[ExcludeFromCodeCoverage] +internal class ApplicationInsightsConfig : IApplicationInsightsConfig +{ + public string ConnectionString { get; set; } = string.Empty; +} + +[ExcludeFromCodeCoverage] +internal class ContentfulConfig: IContentfulConfig +{ + public string DeliveryKey { get; set; } = string.Empty; + public string Environment { get; set; } = string.Empty; + public string SpaceId { get; set; } = string.Empty; +} + +[ExcludeFromCodeCoverage] +internal class SearchIndexingConfig: ISearchIndexingConfig +{ + public string ApiKey { get; set; } = string.Empty; + public int BatchSize { get; set; } = 25; + public string Endpoint { get; set; } = string.Empty; + public string IndexName { get; set; } = string.Empty; + public bool RecreateIndex { get; set; } = false; +} + +[ExcludeFromCodeCoverage] +internal class ApplicationConfiguration: IApplicationConfiguration +{ + public IApplicationInsightsConfig ApplicationInsights { get; set; } = new ApplicationInsightsConfig(); + public string ApplicationVersion { get; set; } = string.Empty; + public IContentfulConfig Contentful { get; set; } = new ContentfulConfig(); + public ISearchIndexingConfig SearchIndexing { get; set; } = new SearchIndexingConfig(); +} \ No newline at end of file diff --git a/src/Childrens-Social-Care-CPD-Indexer/Childrens-Social-Care-CPD-Indexer.csproj b/src/Childrens-Social-Care-CPD-Indexer/Childrens-Social-Care-CPD-Indexer.csproj index 5c81250..7ce2916 100644 --- a/src/Childrens-Social-Care-CPD-Indexer/Childrens-Social-Care-CPD-Indexer.csproj +++ b/src/Childrens-Social-Care-CPD-Indexer/Childrens-Social-Care-CPD-Indexer.csproj @@ -14,11 +14,13 @@ + + - + diff --git a/src/Childrens-Social-Care-CPD-Indexer/Core/IResourcesIndexerConfig.cs b/src/Childrens-Social-Care-CPD-Indexer/Core/IResourcesIndexerConfig.cs deleted file mode 100644 index b83e033..0000000 --- a/src/Childrens-Social-Care-CPD-Indexer/Core/IResourcesIndexerConfig.cs +++ /dev/null @@ -1,15 +0,0 @@ -namespace Childrens_Social_Care_CPD_Indexer.Core; - -public interface IResourcesIndexerConfig -{ - string ApiKey { get; } - string AppInsightsConnectionString { get; } - string ApplicationVersion { get; } - int BatchSize { get; } - string Endpoint { get; } - string IndexName { get; } - string ContentfulApiKey { get; } - string ContentfulEnvironmentId { get; } - string ContentfulSpaceId { get; } - bool RecreateIndex { get; } -} \ No newline at end of file diff --git a/src/Childrens-Social-Care-CPD-Indexer/Program.cs b/src/Childrens-Social-Care-CPD-Indexer/Program.cs index e61132b..2587dd6 100644 --- a/src/Childrens-Social-Care-CPD-Indexer/Program.cs +++ b/src/Childrens-Social-Care-CPD-Indexer/Program.cs @@ -7,17 +7,27 @@ using Microsoft.ApplicationInsights.WorkerService; using Microsoft.ApplicationInsights; using System.Diagnostics.CodeAnalysis; +using Azure.Identity; var builder = Host.CreateApplicationBuilder(args); -builder.Services.AddTransient(); -var config = new ResourcesIndexerConfig(builder.Configuration); +// Configuration +var applicationConfiguration = new ApplicationConfiguration(); +if (!builder.Configuration.GetValue("LOCAL_ENVIRONMENT")) +{ + var keyVaultUri = new Uri($"https://{builder.Configuration.GetValue("CPD_KEY_VAULT_NAME") ?? string.Empty}.vault.azure.net/"); + builder.Configuration.AddAzureKeyVault(keyVaultUri, new DefaultAzureCredential()); +} + +builder.Configuration.Bind(builder.Configuration.GetValue("CPD_CONFIG_SECTION_NAME") ?? string.Empty, applicationConfiguration); +builder.Services.AddSingleton(applicationConfiguration); // Logging + var options = new ApplicationInsightsServiceOptions() { - ApplicationVersion = config.ApplicationVersion, - ConnectionString = config.AppInsightsConnectionString, + ApplicationVersion = applicationConfiguration.ApplicationVersion, + ConnectionString = applicationConfiguration.ApplicationInsights.ConnectionString, }; builder.Services.AddApplicationInsightsTelemetryWorkerService(options); @@ -26,12 +36,12 @@ builder.Services.AddTransient(); builder.Services.AddTransient(servicesProvider => { var httpClient = servicesProvider.GetRequiredService(); - var resourcesIndexerConfig = servicesProvider.GetRequiredService(); + var applicationConfiguration = servicesProvider.GetRequiredService(); var contentfulOptions = new ContentfulOptions() { - DeliveryApiKey = resourcesIndexerConfig.ContentfulApiKey, - SpaceId = resourcesIndexerConfig.ContentfulSpaceId, - Environment = resourcesIndexerConfig.ContentfulEnvironmentId + DeliveryApiKey = applicationConfiguration.Contentful.DeliveryKey, + SpaceId = applicationConfiguration.Contentful.SpaceId, + Environment = applicationConfiguration.Contentful.Environment }; return new ContentfulClient(httpClient, contentfulOptions); }); @@ -39,10 +49,10 @@ builder.Services.AddTransient(); builder.Services.AddTransient(servicesProvider => { var logger = servicesProvider.GetRequiredService>(); - var config = servicesProvider.GetRequiredService(); + var applicationConfiguration = servicesProvider.GetRequiredService(); var documentFetcher = servicesProvider.GetRequiredService(); - var searchEndpointUri = new Uri(config.Endpoint); - var searchIndexClient = new SearchIndexClient(searchEndpointUri, new AzureKeyCredential(config.ApiKey)); + var searchEndpointUri = new Uri(applicationConfiguration.SearchIndexing.Endpoint); + var searchIndexClient = new SearchIndexClient(searchEndpointUri, new AzureKeyCredential(applicationConfiguration.SearchIndexing.ApiKey)); var telemtryClient = servicesProvider.GetRequiredService(); return new ResourcesIndexer(searchIndexClient, documentFetcher, logger, telemtryClient); }); diff --git a/src/Childrens-Social-Care-CPD-Indexer/ResourcesIndexerConfig.cs b/src/Childrens-Social-Care-CPD-Indexer/ResourcesIndexerConfig.cs deleted file mode 100644 index 1c74e9e..0000000 --- a/src/Childrens-Social-Care-CPD-Indexer/ResourcesIndexerConfig.cs +++ /dev/null @@ -1,18 +0,0 @@ -using Childrens_Social_Care_CPD_Indexer.Core; -using Microsoft.Extensions.Configuration; - -namespace Childrens_Social_Care_CPD_Indexer; - -internal class ResourcesIndexerConfig(IConfiguration configuration) : IResourcesIndexerConfig -{ - public string ApiKey => configuration.GetValue("CPD_SEARCH_API_KEY", string.Empty)!; - public string AppInsightsConnectionString => configuration.GetValue("CPD_INSTRUMENTATION_CONNECTIONSTRING", string.Empty)!; - public string ApplicationVersion => configuration.GetValue("VCS-TAG", string.Empty)!; - public int BatchSize => configuration.GetValue("CPD_SEARCH_BATCH_SIZE", 20); - public string Endpoint => configuration.GetValue("CPD_SEARCH_ENDPOINT", string.Empty)!; - public string IndexName => configuration.GetValue("CPD_SEARCH_INDEX_NAME", string.Empty)!; - public string ContentfulApiKey => configuration.GetValue("CPD_DELIVERY_KEY", string.Empty)!; - public string ContentfulEnvironmentId => configuration.GetValue("CPD_CONTENTFUL_ENVIRONMENT", string.Empty)!; - public string ContentfulSpaceId => configuration.GetValue("CPD_SPACE_ID", string.Empty)!; - public bool RecreateIndex => configuration.GetValue("CPD_SEARCH_RECREATE_INDEX_ON_REBUILD", true); -} diff --git a/src/Childrens-Social-Care-CPD-Indexer/Worker.cs b/src/Childrens-Social-Care-CPD-Indexer/Worker.cs index 9db030e..fe1bdbd 100644 --- a/src/Childrens-Social-Care-CPD-Indexer/Worker.cs +++ b/src/Childrens-Social-Care-CPD-Indexer/Worker.cs @@ -6,15 +6,15 @@ public class Worker : BackgroundService { private readonly ILogger _logger; private readonly IResourcesIndexer _resourcesIndexer; - private readonly IResourcesIndexerConfig _config; + private readonly IApplicationConfiguration _config; private readonly IHostApplicationLifetime _hostApplicationLifetime; - public Worker(ILogger logger, IResourcesIndexer resourcesIndexer, IResourcesIndexerConfig config, IHostApplicationLifetime hostApplicationLifetime) + public Worker(ILogger logger, IResourcesIndexer resourcesIndexer, IApplicationConfiguration applicationConfiguration, IHostApplicationLifetime hostApplicationLifetime) { _logger = logger; _resourcesIndexer = resourcesIndexer; - _config = config; _hostApplicationLifetime = hostApplicationLifetime; + _config = applicationConfiguration; } protected override async Task ExecuteAsync(CancellationToken stoppingToken) => @@ -25,12 +25,12 @@ private async Task DoWork(CancellationToken stoppingToken) _logger.LogInformation("Indexing started at: {startTime}", DateTime.Now); try { - if (_config.RecreateIndex) + if (_config.SearchIndexing.RecreateIndex) { - await _resourcesIndexer.DeleteIndexAsync(_config.IndexName, stoppingToken); + await _resourcesIndexer.DeleteIndexAsync(_config.SearchIndexing.IndexName, stoppingToken); } - await _resourcesIndexer.CreateIndexAsync(_config.IndexName, stoppingToken); - await _resourcesIndexer.PopulateIndexAsync(_config.IndexName, _config.BatchSize, stoppingToken); + await _resourcesIndexer.CreateIndexAsync(_config.SearchIndexing.IndexName, stoppingToken); + await _resourcesIndexer.PopulateIndexAsync(_config.SearchIndexing.IndexName, _config.SearchIndexing.BatchSize, stoppingToken); } catch (Exception ex) From 0413a78061ffb1cf323198118d66bda1c85ecfcc Mon Sep 17 00:00:00 2001 From: killij <51908793+killij@users.noreply.github.com> Date: Fri, 19 Jan 2024 13:38:21 +0000 Subject: [PATCH 21/23] fix: Fix logging by flushing the app insights buffer (#26) --- .../WorkerTests.cs | 14 +++++++- .../Worker.cs | 34 ++++++------------- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/src/Childrens-Social-Care-CPD-Indexer.Tests/WorkerTests.cs b/src/Childrens-Social-Care-CPD-Indexer.Tests/WorkerTests.cs index 1a663c6..1bbab38 100644 --- a/src/Childrens-Social-Care-CPD-Indexer.Tests/WorkerTests.cs +++ b/src/Childrens-Social-Care-CPD-Indexer.Tests/WorkerTests.cs @@ -1,4 +1,7 @@ using Childrens_Social_Care_CPD_Indexer.Core; +using Microsoft.ApplicationInsights.Channel; +using Microsoft.ApplicationInsights.Extensibility; +using Microsoft.ApplicationInsights; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using NSubstitute.ExceptionExtensions; @@ -11,6 +14,7 @@ public class WorkerTests private IApplicationConfiguration _config; private IResourcesIndexer _indexer; private IHostApplicationLifetime _hostingApplicationLifetime; + private TelemetryClient _telemetryClient; private Worker _sut; [SetUp] @@ -21,7 +25,15 @@ public void Setup() _indexer = Substitute.For(); _hostingApplicationLifetime = Substitute.For(); - _sut = new Worker(_logger, _indexer, _config, _hostingApplicationLifetime); + var configuration = new TelemetryConfiguration(); + var sendItems = new List(); + var channel = Substitute.For(); + configuration.TelemetryChannel = channel; + configuration.ConnectionString = "InstrumentationKey={Guid.NewGuid()};"; + configuration.TelemetryInitializers.Add(new OperationCorrelationTelemetryInitializer()); + _telemetryClient = new TelemetryClient(configuration); + + _sut = new Worker(_logger, _indexer, _config, _hostingApplicationLifetime, _telemetryClient); } [TearDown] diff --git a/src/Childrens-Social-Care-CPD-Indexer/Worker.cs b/src/Childrens-Social-Care-CPD-Indexer/Worker.cs index fe1bdbd..ae19c8a 100644 --- a/src/Childrens-Social-Care-CPD-Indexer/Worker.cs +++ b/src/Childrens-Social-Care-CPD-Indexer/Worker.cs @@ -1,45 +1,33 @@ using Childrens_Social_Care_CPD_Indexer.Core; +using Microsoft.ApplicationInsights; namespace Childrens_Social_Care_CPD_Indexer; -public class Worker : BackgroundService +public class Worker(ILogger logger, IResourcesIndexer resourcesIndexer, IApplicationConfiguration applicationConfiguration, IHostApplicationLifetime hostApplicationLifetime, TelemetryClient telemetryClient) : BackgroundService { - private readonly ILogger _logger; - private readonly IResourcesIndexer _resourcesIndexer; - private readonly IApplicationConfiguration _config; - private readonly IHostApplicationLifetime _hostApplicationLifetime; - - public Worker(ILogger logger, IResourcesIndexer resourcesIndexer, IApplicationConfiguration applicationConfiguration, IHostApplicationLifetime hostApplicationLifetime) - { - _logger = logger; - _resourcesIndexer = resourcesIndexer; - _hostApplicationLifetime = hostApplicationLifetime; - _config = applicationConfiguration; - } - - protected override async Task ExecuteAsync(CancellationToken stoppingToken) => - await DoWork(stoppingToken).ContinueWith(task => _hostApplicationLifetime.StopApplication(), stoppingToken); + protected override async Task ExecuteAsync(CancellationToken stoppingToken) => await DoWork(stoppingToken).ContinueWith(task => hostApplicationLifetime.StopApplication(), stoppingToken); private async Task DoWork(CancellationToken stoppingToken) { - _logger.LogInformation("Indexing started at: {startTime}", DateTime.Now); + logger.LogInformation("Indexing started at: {startTime}", DateTime.Now); try { - if (_config.SearchIndexing.RecreateIndex) + if (applicationConfiguration.SearchIndexing.RecreateIndex) { - await _resourcesIndexer.DeleteIndexAsync(_config.SearchIndexing.IndexName, stoppingToken); + await resourcesIndexer.DeleteIndexAsync(applicationConfiguration.SearchIndexing.IndexName, stoppingToken); } - await _resourcesIndexer.CreateIndexAsync(_config.SearchIndexing.IndexName, stoppingToken); - await _resourcesIndexer.PopulateIndexAsync(_config.SearchIndexing.IndexName, _config.SearchIndexing.BatchSize, stoppingToken); + await resourcesIndexer.CreateIndexAsync(applicationConfiguration.SearchIndexing.IndexName, stoppingToken); + await resourcesIndexer.PopulateIndexAsync(applicationConfiguration.SearchIndexing.IndexName, applicationConfiguration.SearchIndexing.BatchSize, stoppingToken); } catch (Exception ex) { - _logger.LogError(ex, "Unhandled exception occured"); + logger.LogError(ex, "Unhandled exception occured"); } finally { - _logger.LogInformation("Indexing finished at: {finishTime}", DateTime.Now); + logger.LogInformation("Indexing finished at: {finishTime}", DateTime.Now); + await telemetryClient.FlushAsync(stoppingToken); } } } From 243922a63a5588bf71254c4d8e7d5decd35dc831 Mon Sep 17 00:00:00 2001 From: killij <51908793+killij@users.noreply.github.com> Date: Tue, 23 Jan 2024 08:43:29 +0000 Subject: [PATCH 22/23] feat: Make the Title field searchable (#27) --- src/Childrens-Social-Care-CPD-Indexer/Core/CpdDocument.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Childrens-Social-Care-CPD-Indexer/Core/CpdDocument.cs b/src/Childrens-Social-Care-CPD-Indexer/Core/CpdDocument.cs index 6ba03cf..b43c1ae 100644 --- a/src/Childrens-Social-Care-CPD-Indexer/Core/CpdDocument.cs +++ b/src/Childrens-Social-Care-CPD-Indexer/Core/CpdDocument.cs @@ -11,7 +11,7 @@ internal partial class CpdDocument(string id) [SimpleField(IsKey = true)] public string? Id { get; set; } = Base64UrlEncoder.Encode(id); - [SimpleField()] + [SearchableField(AnalyzerName = LexicalAnalyzerName.Values.EnLucene)] public string? Title { get; set; } [SimpleField(IsFilterable = true, IsFacetable = true)] From 9766e70e637befbc25f43345aa1b0e0104bb1764 Mon Sep 17 00:00:00 2001 From: killij <51908793+killij@users.noreply.github.com> Date: Wed, 14 Feb 2024 09:04:34 +0000 Subject: [PATCH 23/23] fix: Remove the environment aspect of the configuration (#28) --- src/Childrens-Social-Care-CPD-Indexer/Program.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Childrens-Social-Care-CPD-Indexer/Program.cs b/src/Childrens-Social-Care-CPD-Indexer/Program.cs index 2587dd6..ce85d1f 100644 --- a/src/Childrens-Social-Care-CPD-Indexer/Program.cs +++ b/src/Childrens-Social-Care-CPD-Indexer/Program.cs @@ -20,7 +20,7 @@ builder.Configuration.AddAzureKeyVault(keyVaultUri, new DefaultAzureCredential()); } -builder.Configuration.Bind(builder.Configuration.GetValue("CPD_CONFIG_SECTION_NAME") ?? string.Empty, applicationConfiguration); +builder.Configuration.Bind("IndexingConfig", applicationConfiguration); builder.Services.AddSingleton(applicationConfiguration); // Logging