Adds support for ECR delete messages

Defra.Cdp.Backend.Api.Tests/Services/Aws/EcrEventListenerTest.cs

@@ -0,0 +1,118 @@
+using Defra.Cdp.Backend.Api.Models;
+using Defra.Cdp.Backend.Api.Services.Aws;
+using Defra.Cdp.Backend.Api.Services.TenantArtifacts;
+using Microsoft.Extensions.Logging;
+using NSubstitute;
+
+namespace Defra.Cdp.Backend.Api.Tests.Services.Aws;
+
+public class EcrEventListenerTest
+{
+
+    IArtifactScanner docker = Substitute.For<IArtifactScanner>();
+    IDeployableArtifactsService artifacts = Substitute.For<IDeployableArtifactsService>();
+    ILogger<EcrEventListener> logger = ConsoleLogger.CreateLogger<EcrEventListener>();
+
+    [Fact]
+    public async Task TestInvalidMessage()
+    {
+        var handler = new EcrMessageHandler(docker, artifacts, logger);
+        var act = () => handler.Handle("123", "invalid", new CancellationToken());
+
+        await Assert.ThrowsAsync<System.Text.Json.JsonException>(act);
+        await docker.DidNotReceive().ScanImage(Arg.Any<string>(), Arg.Any<string>(), Arg.Any<CancellationToken>());
+        await artifacts.DidNotReceive().RemoveAsync(Arg.Any<string>(), Arg.Any<string>(), Arg.Any<CancellationToken>());
+    }
+
+    [Fact]
+    public async Task TestValidPushMessage()
+    {
+        var message = """
+                      {
+                          "version": "0",
+                          "id": "13cde686-328b-6117-af20-0e5566167482",
+                          "detail-type": "ECR Image Action",
+                          "source": "aws.ecr",
+                          "account": "123456789012",
+                          "time": "2019-11-16T01:54:34Z",
+                          "region": "us-west-2",
+                          "resources": [],
+                          "detail": {
+                              "result": "SUCCESS",
+                              "repository-name": "my-repository-name",
+                              "image-digest": "sha256:7f5b2640fe6fb4f46592dfd3410c4a79dac4f89e4782432e0378abcd1234",
+                              "action-type": "PUSH",
+                              "image-tag": "0.1.0"
+                          }
+                      }
+                      """;
+        docker.ScanImage("my-repository-name", "0.1.0", Arg.Any<CancellationToken>()).Returns(new ArtifactScannerResult(new DeployableArtifact()
+        {
+            Repo = "my-repository-name",
+            Tag = "0.1.0",
+            Sha256 = "sha256:7f5b2640fe6fb4f46592dfd3410c4a79dac4f89e4782432e0378abcd1234"
+        }));
+        var handler = new EcrMessageHandler(docker, artifacts, logger);
+        await handler.Handle("123", message, new CancellationToken());
+        await docker.Received().ScanImage(Arg.Is("my-repository-name"), Arg.Is("0.1.0"), Arg.Any<CancellationToken>());
+        await artifacts.DidNotReceive().RemoveAsync(Arg.Any<string>(), Arg.Any<string>(), Arg.Any<CancellationToken>());
+    }
+
+    [Fact]
+    public async Task TestNonSemverPushMessage()
+    {
+        var message = """
+                      {
+                          "version": "0",
+                          "id": "13cde686-328b-6117-af20-0e5566167482",
+                          "detail-type": "ECR Image Action",
+                          "source": "aws.ecr",
+                          "account": "123456789012",
+                          "time": "2019-11-16T01:54:34Z",
+                          "region": "us-west-2",
+                          "resources": [],
+                          "detail": {
+                              "result": "SUCCESS",
+                              "repository-name": "my-repository-name",
+                              "image-digest": "sha256:7f5b2640fe6fb4f46592dfd3410c4a79dac4f89e4782432e0378abcd1234",
+                              "action-type": "PUSH",
+                              "image-tag": "27"
+                          }
+                      }
+                      """;
+        var handler = new EcrMessageHandler(docker, artifacts, logger);
+        await handler.Handle("123", message, new CancellationToken());
+        await docker.DidNotReceive().ScanImage(Arg.Is("my-repository-name"), Arg.Is("0.1.0"), Arg.Any<CancellationToken>());
+        await artifacts.DidNotReceive().RemoveAsync(Arg.Any<string>(), Arg.Any<string>(), Arg.Any<CancellationToken>());
+    }
+
+
+    [Fact]
+    public async Task TestValidDeleteMessage()
+    {
+        var message = """
+                      {
+                          "version": "0",
+                          "id": "13cde686-328b-6117-af20-0e5566167482",
+                          "detail-type": "ECR Image Action",
+                          "source": "aws.ecr",
+                          "account": "123456789012",
+                          "time": "2019-11-16T01:54:34Z",
+                          "region": "us-west-2",
+                          "resources": [],
+                          "detail": {
+                              "result": "SUCCESS",
+                              "repository-name": "my-repository-name",
+                              "image-digest": "sha256:7f5b2640fe6fb4f46592dfd3410c4a79dac4f89e4782432e0378abcd1234",
+                              "action-type": "DELETE",
+                              "image-tag": "0.1.0"
+                          }
+                      }
+                      """;
+
+        var handler = new EcrMessageHandler(docker, artifacts, logger);
+        await handler.Handle("123", message, new CancellationToken());
+        await docker.DidNotReceive().ScanImage(Arg.Any<string>(), Arg.Any<string>(), Arg.Any<CancellationToken>());
+        await artifacts.Received().RemoveAsync(Arg.Is("my-repository-name"), Arg.Is("0.1.0"),  Arg.Any<CancellationToken>());
+    }
+}

Defra.Cdp.Backend.Api/Program.cs

@@ -155,6 +155,7 @@
 builder.Services.AddSingleton<IEnvironmentLookup, EnvironmentLookup>();
 builder.Services.AddSingleton<EcrEventListener>();
 builder.Services.AddSingleton<EcsEventListener>();
+builder.Services.AddSingleton<EcrMessageHandler>();
 builder.Services.AddSingleton<TemplatesFromConfig>();
 builder.Services.AddSingleton<ITemplatesService, TemplatesService>();
 builder.Services.AddSingleton<ITestRunService, TestRunService>();

Defra.Cdp.Backend.Api/Services/Aws/EcrEventListener.cs

@@ -10,15 +10,12 @@ namespace Defra.Cdp.Backend.Api.Services.Aws;
 
 public class EcrEventListener(
     IAmazonSQS sqs,
-    IArtifactScanner docker,
+    EcrMessageHandler handler,
     IOptions<EcrEventListenerOptions> config,
     IEcrEventsService ecrEventService,
     ILogger<EcrEventListener> logger)
     : SqsListener(sqs, config.Value.QueueUrl, logger)
 {
-    private readonly EcrEventListenerOptions _options = config.Value;
-    private readonly IAmazonSQS _sqs = sqs;
-
     protected override async Task HandleMessageAsync(Message message, CancellationToken cancellationToken)
     {
         logger.LogInformation("Received message: {MessageId}", message.MessageId);
@@ -34,25 +31,21 @@ protected override async Task HandleMessageAsync(Message message, CancellationTo
 
         try
         {
-            var result = await HandleEcrMessage(message.MessageId, message.Body, cancellationToken);
-            if (result is { Success: true, Artifact: not null })
-                logger.LogInformation(
-                    "Processed {MsgMessageId}, image ${ResultSha256} ({ResultRepo}:{ResultTag}) scanned ok",
-                    message.MessageId, result.Artifact.Sha256, result.Artifact.Repo, result.Artifact.Tag);
-            else
-                logger.LogInformation("Skipping processing of {MessageId}, {Error}", message.MessageId, result.Error);
+            await handler.Handle(message.MessageId, message.Body, cancellationToken);
         }
         catch (Exception ex)
         {
-            logger.LogError(ex, "Failed to scan image for event {MessageId}", message.MessageId);
+            logger.LogError(ex, "Failed to process ECR message for event {MessageId}, {Error}", message.MessageId, ex.Message);
         }
-
-        // TODO: better error detection to decide if we delete, dead letter or retry...
-        await _sqs.DeleteMessageAsync(_options.QueueUrl, message.ReceiptHandle, cancellationToken);
     }
-
-
-    private async Task<ArtifactScannerResult> HandleEcrMessage(string id, string body, CancellationToken cancellationToken)
+}
+
+public class EcrMessageHandler(
+        IArtifactScanner docker,
+        IDeployableArtifactsService artifactsService,
+        ILogger<EcrEventListener> logger)
+{
+    public async Task Handle(string id, string body, CancellationToken cancellationToken)
     {
         logger.LogInformation("Starting processing ECR Event {Id}", id);
         // AWS JSON messages are sent in with their " escaped (\"), in order to parse, they must be unescaped
@@ -62,23 +55,31 @@ private async Task<ArtifactScannerResult> HandleEcrMessage(string id, string bod
         if (ecrEvent?.Detail == null)
         {
             logger.LogInformation("Not processing {Id}, failed to process json", id);
-            return ArtifactScannerResult.Failure($"Not processing {id}, failed to process json");
+            return;
         }
 
         if (ecrEvent.Detail.Result != "SUCCESS")
-            return ArtifactScannerResult.Failure($"Not processing {id}, result is not a SUCCESS");
-
-        if (ecrEvent.Detail.ActionType != "PUSH")
-            return ArtifactScannerResult.Failure($"Not processing {id}, message is not a PUSH event");
-
-        if (!SemVer.IsSemVer(ecrEvent.Detail.ImageTag))
         {
-            logger.LogInformation("Not processing {Id}, tag [{ImageTag}] is not semver", id, ecrEvent.Detail.ImageTag);
-            // TODO: have a better return type that can indicate why it wasn't scanned.
-            return ArtifactScannerResult.Failure(
-                $"Not processing {id}, tag [{ecrEvent.Detail.ImageTag}] is not semver");
+            logger.LogInformation("Processing ECR Event {Id}, failed to process json", id);
+            return;
         }
 
-        return await docker.ScanImage(ecrEvent.Detail.RepositoryName, ecrEvent.Detail.ImageTag, cancellationToken);
+        switch (ecrEvent.Detail.ActionType)
+        {
+            case "PUSH" when !SemVer.IsSemVer(ecrEvent.Detail.ImageTag):
+                logger.LogInformation("Not processing {Id}, tag [{ImageTag}] is not semver", id, ecrEvent.Detail.ImageTag);
+                break;
+            case "PUSH":
+                var result = await docker.ScanImage(ecrEvent.Detail.RepositoryName, ecrEvent.Detail.ImageTag, cancellationToken);
+                logger.LogInformation("Scanned {Sha256} ({Repo}:{Tag}) {Result}", result.Artifact?.Sha256, result.Artifact?.Repo, result.Artifact?.Tag, result.Success ? "OK" : "FAILED");
+                break;
+            case "DELETE":
+                var deleted = await artifactsService.RemoveAsync(ecrEvent.Detail.RepositoryName, ecrEvent.Detail.ImageTag, cancellationToken);
+                logger.LogInformation("Deleted {Sha256} ({Repo}:{Tag}) {Result}", ecrEvent.Detail.ImageDigest, ecrEvent.Detail.RepositoryName, ecrEvent.Detail.ImageTag, deleted ? "OK" : "FAILED");
+                break;
+            default:
+                logger.LogInformation("Not processing {id}, message is not a PUSH or DELETE event", id);
+                break;
+        }
     }
-}
+}

Defra.Cdp.Backend.Api/Services/Aws/EcrEventService.cs

@@ -9,16 +9,11 @@ public interface IEcrEventsService
     Task SaveMessage(string id, string body, CancellationToken cancellationToken);
 }
 
-public class EcrEventsService : MongoService<EcrEventCopy>, IEcrEventsService
+public class EcrEventsService(IMongoDbClientFactory connectionFactory, ILoggerFactory loggerFactory)
+    : MongoService<EcrEventCopy>(connectionFactory, CollectionName, loggerFactory), IEcrEventsService
 {
     private const string CollectionName = "ecrevents";
 
-    public EcrEventsService(IMongoDbClientFactory connectionFactory, ILoggerFactory loggerFactory) : base(
-        connectionFactory,
-        CollectionName, loggerFactory)
-    {
-    }
-
     public async Task SaveMessage(string id, string body, CancellationToken cancellationToken)
     {
         await Collection.InsertOneAsync(new EcrEventCopy(id, new DateTimeOffset(), body),

Defra.Cdp.Backend.Api/Services/Aws/SqsListener.cs

@@ -49,7 +49,6 @@ public async Task ReadAsync(CancellationToken cancellationToken)
                     {
                         logger.LogError(message.Body);
                         logger.LogError(exception.Message);
-                        // TODO: support Dead Letter Queue 
                     }
 
                     var deleteRequest = new DeleteMessageRequest

Defra.Cdp.Backend.Api/Services/TenantArtifacts/ArtifactScanner.cs

@@ -17,7 +17,7 @@ public class ArtifactScannerResult
     public readonly string Error;
     public readonly bool Success;
 
-    public ArtifactScannerResult(DeployableArtifact artifact)
+    public ArtifactScannerResult(DeployableArtifact? artifact)
     {
         Artifact = artifact;
         Success = true;

Defra.Cdp.Backend.Api/Services/TenantArtifacts/DeployableArtifactService.cs

@@ -13,6 +13,8 @@ public interface IDeployableArtifactsService
     Task CreatePlaceholderAsync(string serviceName, string githubUrl, ArtifactRunMode runMode,
         CancellationToken cancellationToken);
 
+    Task<bool> RemoveAsync(string service, string tag, CancellationToken cancellationToken);
+
     Task<List<DeployableArtifact>> FindAll(CancellationToken cancellationToken);
     Task<List<DeployableArtifact>> FindAll(string repo, CancellationToken cancellationToken);
 
@@ -55,6 +57,12 @@ public class DeployableArtifactsService(IMongoDbClientFactory connectionFactory,
             .FirstOrDefaultAsync(cancellationToken);
     }
 
+    public async Task<bool> RemoveAsync(string service, string tag, CancellationToken cancellationToken)
+    {
+        var result = await Collection.DeleteOneAsync(d => d.Repo == service && d.Tag == tag, cancellationToken);
+        return result.DeletedCount == 1;
+    }
+
     public async Task<List<DeployableArtifact>> FindAll(CancellationToken cancellationToken)
     {
         return await Collection.Find(FilterDefinition<DeployableArtifact>.Empty).ToListAsync(cancellationToken);