Skip to content

Commit

Permalink
Merge pull request #592 from dlcs/feature/customer-stoarge
Browse files Browse the repository at this point in the history 
Modifying customer storage to take into account the size of the previous asset
  • Loading branch information
@JackLewis-digirati
JackLewis-digirati authored Aug 16, 2023
2 parents 0dcb1b5 + 8e812c9 commit 50edd2c
Show file tree
Hide file tree
Showing 20 changed files with 276 additions and 89 deletions.
2 changes: 1 addition & 1 deletion src/protagonist/API.Tests/Integration/PolicyTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,7 @@ public async Task Get_StoragePolicies_200()
response.StatusCode.Should().Be(HttpStatusCode.OK);

var model = await response.ReadAsHydraResponseAsync<HydraCollection<StoragePolicy>>();
model.Members.Should().HaveCount(2);
model.Members.Should().HaveCount(3);
}

[Fact]
Expand Down
4 changes: 2 additions & 2 deletions src/protagonist/DLCS.Model/Storage/AssetStorageMetric.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,6 @@ public bool CanStoreAsset(int assetCount = 1) =>
/// <summary>
/// Check if there is allowance to store asset based on size
/// </summary>
public bool CanStoreAssetSize(long proposedNewSize) =>
CustomerStorage.TotalSizeOfStoredImages + proposedNewSize <= Policy.MaximumTotalSizeOfStoredImages;
public bool CanStoreAssetSize(long proposedNewSize, long oldFileSize) =>
(CustomerStorage.TotalSizeOfStoredImages - oldFileSize) + proposedNewSize <= Policy.MaximumTotalSizeOfStoredImages;
}
12 changes: 6 additions & 6 deletions src/protagonist/Engine.Tests/Ingest/File/FileChannelWorkerTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ public async Task Ingest_NoOp_IfOptimisedStrategy()
// Assert
result.Should().Be(IngestResultStatus.Success);
A.CallTo(() =>
assetToS3.CopyOriginToStorage(A<ObjectInBucket>._, A<Asset>._, A<bool>._, cos, A<CancellationToken>._))
assetToS3.CopyOriginToStorage(A<ObjectInBucket>._, A<IngestionContext>._, A<bool>._, cos, A<CancellationToken>._))
.MustNotHaveHappened();
}

Expand All @@ -56,7 +56,7 @@ public async Task Ingest_CopiesFileToStorage_SetsImageStorage_AndStoredObject()
.Returns(destination);

A.CallTo(() =>
assetToS3.CopyOriginToStorage(destination, context.Asset, true, cos, A<CancellationToken>._))
assetToS3.CopyOriginToStorage(destination, context, true, cos, A<CancellationToken>._))
.Returns(new AssetFromOrigin(context.AssetId, 1234L, "anywhere", "application/docx"));

// Act
Expand All @@ -81,7 +81,7 @@ public async Task Ingest_CopiesFileToStorage_IncrementsImageStorage_AndStoredObj
.Returns(destination);

A.CallTo(() =>
assetToS3.CopyOriginToStorage(destination, context.Asset, true, cos, A<CancellationToken>._))
assetToS3.CopyOriginToStorage(destination, context, true, cos, A<CancellationToken>._))
.Returns(new AssetFromOrigin(context.AssetId, 1234L, "anywhere", "application/docx"));

// Act
Expand All @@ -92,7 +92,7 @@ public async Task Ingest_CopiesFileToStorage_IncrementsImageStorage_AndStoredObj
context.StoredObjects.Should().ContainKey(destination).WhoseValue.Should().Be(1234L);
result.Should().Be(IngestResultStatus.Success);
}

[Fact]
public async Task Ingest_ReturnsErrorIfCopyExceedStorageLimit()
{
Expand All @@ -107,7 +107,7 @@ public async Task Ingest_ReturnsErrorIfCopyExceedStorageLimit()
var assetFromOrigin = new AssetFromOrigin(context.AssetId, 1234L, "anywhere", "application/docx");
assetFromOrigin.FileTooLarge();
A.CallTo(() =>
assetToS3.CopyOriginToStorage(destination, context.Asset, true, cos, A<CancellationToken>._))
assetToS3.CopyOriginToStorage(destination, context, true, cos, A<CancellationToken>._))
.Returns(assetFromOrigin);

// Act
Expand Down Expand Up @@ -135,7 +135,7 @@ public async Task Ingest_CopiesFileToStorage_PassesVerifySizeFalse_IfCustomerExc

// Assert
A.CallTo(() =>
assetToS3.CopyOriginToStorage(destination, context.Asset, false, cos, A<CancellationToken>._))
assetToS3.CopyOriginToStorage(destination, context, false, cos, A<CancellationToken>._))
.MustHaveHappened();
result.Should().Be(IngestResultStatus.Success);
}
Expand Down
26 changes: 24 additions & 2 deletions src/protagonist/Engine.Tests/Ingest/IngestExecutorTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,12 +13,14 @@ public class IngestExecutorTests
private readonly IEngineAssetRepository repo;
private readonly IngestExecutor sut;
private readonly CustomerOriginStrategy customerOriginStrategy = new();
private readonly IAssetIngestorSizeCheck assetSizeCheck;

public IngestExecutorTests()
{
workerBuilder = A.Fake<IWorkerBuilder>();
repo = A.Fake<IEngineAssetRepository>();
sut = new IngestExecutor(workerBuilder, repo, new NullLogger<IngestExecutor>());
assetSizeCheck = A.Fake<IAssetIngestorSizeCheck>();
sut = new IngestExecutor(workerBuilder, repo, assetSizeCheck, new NullLogger<IngestExecutor>());
}

[Fact]
Expand All @@ -35,6 +37,26 @@ public async Task IngestAsset_HandlesNoWorkers()
repo.UpdateIngestedAsset(asset, A<ImageLocation?>._, A<ImageStorage?>._, true, A<CancellationToken>._))
.MustHaveHappened();
}

[Theory]
[InlineData(true)]
[InlineData(false)]
public async Task IngestAsset_Handles_CanHaveAssetSizeCheck(bool sizeCheck)
{
var asset = new Asset();
A.CallTo(() => workerBuilder.GetWorkers(asset))
.Returns(new[] { new FakeWorker(IngestResultStatus.Success) });
A.CallTo(() => assetSizeCheck.CustomerHasNoStorageCheck(A<int>._))
.Returns(sizeCheck);

// Act
await sut.IngestAsset(asset, customerOriginStrategy);

// Assert
A.CallTo(() =>
repo.UpdateIngestedAsset(asset, A<ImageLocation?>._, A<ImageStorage?>._, true, A<CancellationToken>._))
.MustHaveHappened();
}

[Theory]
[InlineData(IngestResultStatus.Success, IngestResultStatus.Success, IngestResultStatus.Success)]
Expand All @@ -59,7 +81,7 @@ public async Task IngestAsset_Success_ReturnsCorrectStatus(IngestResultStatus fi

// Act
var result = await sut.IngestAsset(asset, customerOriginStrategy);

// Assert
result.Status.Should().Be(overall);
}
Expand Down
24 changes: 16 additions & 8 deletions src/protagonist/Engine.Tests/Ingest/Persistence/AssetToDiskTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
using DLCS.Repository.Strategy;
using DLCS.Repository.Strategy.DependencyInjection;
using DLCS.Repository.Strategy.Utils;
using Engine.Ingest;
using Engine.Ingest.Persistence;
using FakeItEasy;
using Microsoft.Extensions.Logging.Abstractions;
Expand Down Expand Up @@ -40,7 +41,7 @@ public AssetToDiskTests()
public void CopyAssetFromOrigin_Throws_IfDestinationFolderNullOrEmpty(string destinationFolder)
{
// Act
Func<Task> action = () => sut.CopyAssetToLocalDisk(new Asset(), destinationFolder, true, new CustomerOriginStrategy());
Func<Task> action = () => sut.CopyAssetToLocalDisk(new IngestionContext(new Asset()), destinationFolder, true, new CustomerOriginStrategy());

// Assert
action.Should()
Expand All @@ -54,13 +55,14 @@ public void CopyAssetFromOrigin_Throws_IfOriginReturnsNull()
// Arrange
const string origin = "http://test-origin";
var asset = new Asset { Id = AssetId.FromString("/2/1/godzilla") };
var context = new IngestionContext(asset);
var cos = new CustomerOriginStrategy { Strategy = OriginStrategyType.S3Ambient };
A.CallTo(() =>
customerOriginStrategy.LoadAssetFromOrigin(asset.Id, origin, cos, A<CancellationToken>._))
.Returns<OriginResponse?>(null);

// Act
Func<Task> action = () => sut.CopyAssetToLocalDisk(asset, "./here", true, cos);
Func<Task> action = () => sut.CopyAssetToLocalDisk(context, "./here", true, cos);

// Assert
action.Should().ThrowAsync<ApplicationException>();
Expand All @@ -73,12 +75,13 @@ public void CopyAssetFromOrigin_Throws_IfOriginReturnsEmptyStream()
const string origin = "http://test-origin";
var asset = new Asset { Id = AssetId.FromString("/2/1/godzilla"), Origin = origin};
var cos = new CustomerOriginStrategy { Strategy = OriginStrategyType.S3Ambient };
var context = new IngestionContext(asset);
A.CallTo(() =>
customerOriginStrategy.LoadAssetFromOrigin(asset.Id, origin, cos, A<CancellationToken>._))
.Returns(new OriginResponse(Stream.Null));

// Act
Func<Task> action = () => sut.CopyAssetToLocalDisk(asset, "./here", true, cos);
Func<Task> action = () => sut.CopyAssetToLocalDisk(context, "./here", true, cos);

// Assert
action.Should().ThrowAsync<ApplicationException>();
Expand All @@ -93,6 +96,7 @@ public async Task CopyAssetFromOrigin_SavesFileToDisk_IfNoContentLength()
AssetId assetId = AssetId.FromString("2/1/godzilla");
var asset = new Asset(assetId) { Origin = origin };
var cos = new CustomerOriginStrategy { Strategy = OriginStrategyType.S3Ambient };
var context = new IngestionContext(asset);

var responseStream = "{\"foo\":\"bar\"}".ToMemoryStream();
var originResponse = new OriginResponse(responseStream).WithContentType("application/json");
Expand All @@ -106,7 +110,7 @@ public async Task CopyAssetFromOrigin_SavesFileToDisk_IfNoContentLength()
var expectedOutput = Path.Join(".", "2", "1", "godzilla", "godzilla.file");

// Act
var response = await sut.CopyAssetToLocalDisk(asset, destination, false, cos);
var response = await sut.CopyAssetToLocalDisk(context, destination, false, cos);

// Assert
A.CallTo(() => fileSaver.SaveResponseToDisk(A<AssetId>.That.Matches(a => a == assetId),
Expand All @@ -126,6 +130,7 @@ public async Task CopyAssetFromOrigin_SavesFileToDisk_IfContentLength()
const string origin = "http://test-origin";
var assetId = AssetId.FromString("2/1/godzilla1");
var asset = new Asset(assetId) { Origin = origin };
var context = new IngestionContext(asset);

var cos = new CustomerOriginStrategy { Strategy = OriginStrategyType.S3Ambient };

Expand All @@ -143,7 +148,7 @@ public async Task CopyAssetFromOrigin_SavesFileToDisk_IfContentLength()
var expectedOutput = Path.Join(".", "2", "1", "godzilla1", "godzilla1.file");

// Act
var response = await sut.CopyAssetToLocalDisk(asset, destination, false, cos);
var response = await sut.CopyAssetToLocalDisk(context, destination, false, cos);

// Assert
A.CallTo(() => fileSaver.SaveResponseToDisk(A<AssetId>.That.Matches(a => a == assetId),
Expand All @@ -166,6 +171,7 @@ public async Task CopyAssetFromOrigin_SetsExtension_BasedOnFileType(string conte
const string origin = "http://test-origin";
var asset = new Asset { Id = AssetId.FromString("/2/1/godzilla.jp2"), Customer = 2, Space = 1, Origin = origin };
var cos = new CustomerOriginStrategy { Strategy = OriginStrategyType.S3Ambient };
var context = new IngestionContext(asset);

var responseStream = "{\"foo\":\"bar\"}".ToMemoryStream();
var originResponse = new OriginResponse(responseStream)
Expand All @@ -176,7 +182,7 @@ public async Task CopyAssetFromOrigin_SetsExtension_BasedOnFileType(string conte
.Returns(originResponse);

// Act
var response = await sut.CopyAssetToLocalDisk(asset, destination, false, cos);
var response = await sut.CopyAssetToLocalDisk(context, destination, false, cos);

// Assert
response.ContentType.Should().Be(contentType);
Expand All @@ -193,6 +199,7 @@ public async Task CopyAssetFromOrigin_SetsContentType_IfUnknownOrBinary_AssetIdI
const string origin = "http://test-origin";
var asset = new Asset { Id = AssetId.FromString("/2/1/godzilla.jp2"), Customer = 2, Space = 1, Origin = origin };
var cos = new CustomerOriginStrategy { Strategy = OriginStrategyType.S3Ambient };
var context = new IngestionContext(asset);

var responseStream = "{\"foo\":\"bar\"}".ToMemoryStream();
var originResponse = new OriginResponse(responseStream)
Expand All @@ -203,7 +210,7 @@ public async Task CopyAssetFromOrigin_SetsContentType_IfUnknownOrBinary_AssetIdI
.Returns(originResponse);

// Act
var response = await sut.CopyAssetToLocalDisk(asset, destination, false, cos);
var response = await sut.CopyAssetToLocalDisk(context, destination, false, cos);

// Assert
response.ContentType.Should().Be("image/jp2");
Expand All @@ -219,6 +226,7 @@ public async Task CopyAssetFromOrigin_VerifiesFileSize(bool isValid)
const string origin = "http://test-origin";
var asset = new Asset { Id = AssetId.FromString("/2/1/godzilla"), Customer = 2, Space = 1, Origin = origin };
var cos = new CustomerOriginStrategy { Strategy = OriginStrategyType.S3Ambient };
var context = new IngestionContext(asset);

var responseStream = "{\"foo\":\"bar\"}".ToMemoryStream();
var originResponse = new OriginResponse(responseStream).WithContentType("application/json");
Expand All @@ -234,7 +242,7 @@ public async Task CopyAssetFromOrigin_VerifiesFileSize(bool isValid)
});

// Act
var response = await sut.CopyAssetToLocalDisk(asset, destination, true, cos);
var response = await sut.CopyAssetToLocalDisk(context, destination, true, cos);

// Assert
response.FileExceedsAllowance.Should().Be(!isValid);
Expand Down
Loading

0 comments on commit 50edd2c

Please sign in to comment.