From 163b2734e5f0a42ec0b05e44122db85162d0ce0f Mon Sep 17 00:00:00 2001 From: Simon Date: Mon, 9 May 2022 09:04:32 +0200 Subject: [PATCH] Fix upload empty files on remote storage (#3241) Co-authored-by: Simon Dumas --- .../storage/files/FormDataExtractor.scala | 11 +++-- .../storages/operations/SaveFile.scala | 9 ++-- .../operations/disk/DiskStorageSaveFile.scala | 41 ++++++++++--------- .../remote/RemoteDiskStorageSaveFile.scala | 6 +-- .../client/RemoteDiskStorageClient.scala | 13 +++--- .../operations/s3/S3StorageSaveFile.scala | 9 ++-- .../storage/files/FormDataExtractorSpec.scala | 6 +-- .../disk/DiskStorageSaveFileSpec.scala | 10 ++--- .../RemoteStorageSaveAndFetchFileSpec.scala | 10 ++--- .../client/RemoteStorageClientSpec.scala | 8 ++-- .../operations/s3/S3StorageLinkFileSpec.scala | 8 ++-- .../s3/S3StorageSaveAndFetchFileSpec.scala | 19 ++++----- tests/src/test/resources/kg/files/empty | 0 .../resources/kg/storages/statistics.json | 2 +- .../nexus/tests/kg/StorageSpec.scala | 23 +++++++++++ 15 files changed, 92 insertions(+), 83 deletions(-) create mode 100644 tests/src/test/resources/kg/files/empty diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FormDataExtractor.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FormDataExtractor.scala index 2d72b9229b..7540d384c5 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FormDataExtractor.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FormDataExtractor.scala @@ -1,7 +1,7 @@ package ch.epfl.bluebrain.nexus.delta.plugins.storage.files import akka.actor.ActorSystem -import akka.http.scaladsl.model.{EntityStreamSizeException, ExceptionWithErrorInfo, HttpEntity, Multipart} +import akka.http.scaladsl.model._ import akka.http.scaladsl.server._ import akka.http.scaladsl.unmarshalling.Unmarshaller.UnsupportedContentTypeException import akka.http.scaladsl.unmarshalling.{FromEntityUnmarshaller, Unmarshaller} @@ -11,7 +11,6 @@ import ch.epfl.bluebrain.nexus.delta.kernel.utils.UUIDF import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileRejection.{FileTooLarge, InvalidMultipartFieldName, WrappedAkkaRejection} import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{FileDescription, FileRejection} import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri -import ch.epfl.bluebrain.nexus.delta.sdk.AkkaSource import monix.bio.IO import monix.execution.Scheduler @@ -29,14 +28,14 @@ sealed trait FormDataExtractor { * @param storageAvailableSpace * the remaining available space on the storage * @return - * the file description plus the stream of [[ByteString]] with the file content + * the file description plus the entity with the file content */ def apply( id: Iri, entity: HttpEntity, maxFileSize: Long, storageAvailableSpace: Option[Long] - ): IO[FileRejection, (FileDescription, AkkaSource)] + ): IO[FileRejection, (FileDescription, BodyPartEntity)] } object FormDataExtractor { @@ -53,7 +52,7 @@ object FormDataExtractor { entity: HttpEntity, maxFileSize: Long, storageAvailableSpace: Option[Long] - ): IO[FileRejection, (FileDescription, AkkaSource)] = { + ): IO[FileRejection, (FileDescription, BodyPartEntity)] = { val sizeLimit = Math.min(storageAvailableSpace.getOrElse(Long.MaxValue), maxFileSize) IO.deferFuture(um(entity.withSizeLimit(sizeLimit))) .mapError { @@ -76,7 +75,7 @@ object FormDataExtractor { .mapAsync(parallelism = 1) { case part if part.name == fieldName => FileDescription(part.filename.getOrElse("file"), part.entity.contentType).runToFuture.map { desc => - Some(desc -> part.entity.dataBytes) + Some(desc -> part.entity) } case part => part.entity.discardBytes().future.as(None) diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/SaveFile.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/SaveFile.scala index ee899d61e7..7c95617e8e 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/SaveFile.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/SaveFile.scala @@ -1,13 +1,12 @@ package ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations import akka.actor.ActorSystem -import akka.http.scaladsl.model.Uri +import akka.http.scaladsl.model.{BodyPartEntity, Uri} import akka.stream.scaladsl.Sink import akka.util.ByteString import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.Digest.ComputedDigest import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{FileAttributes, FileDescription} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.StoragesConfig.StorageTypeConfig -import ch.epfl.bluebrain.nexus.delta.sdk.AkkaSource import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.{DigestAlgorithm, Storage} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.StorageFileRejection.SaveFileRejection import ch.epfl.bluebrain.nexus.delta.sdk.http.HttpClient @@ -24,10 +23,10 @@ trait SaveFile { * * @param description * the file description - * @param source - * the file stream + * @param entity + * the entity with the file content */ - def apply(description: FileDescription, source: AkkaSource): IO[SaveFileRejection, FileAttributes] + def apply(description: FileDescription, entity: BodyPartEntity): IO[SaveFileRejection, FileAttributes] } object SaveFile { diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskStorageSaveFile.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskStorageSaveFile.scala index 2a8f6633ac..343d9b2b60 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskStorageSaveFile.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskStorageSaveFile.scala @@ -1,11 +1,10 @@ package ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.disk import akka.actor.ActorSystem -import akka.http.scaladsl.model.Uri +import akka.http.scaladsl.model.{BodyPartEntity, Uri} import akka.stream.scaladsl.FileIO import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileAttributes.FileAttributesOrigin.Client import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{FileAttributes, FileDescription} -import ch.epfl.bluebrain.nexus.delta.sdk.AkkaSource import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.Storage.DiskStorage import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.SaveFile import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.SaveFile.{digestSink, intermediateFolders} @@ -15,7 +14,7 @@ import ch.epfl.bluebrain.nexus.delta.plugins.storage.utils.SinkUtils import monix.bio.IO import java.nio.file.StandardOpenOption._ -import java.nio.file.{FileAlreadyExistsException, Files, OpenOption, Path, Paths} +import java.nio.file._ import java.util.UUID import scala.concurrent.Future import scala.util.Try @@ -26,26 +25,28 @@ final class DiskStorageSaveFile(storage: DiskStorage)(implicit as: ActorSystem) private val openOpts: Set[OpenOption] = Set(CREATE_NEW, WRITE) - override def apply(description: FileDescription, source: AkkaSource): IO[SaveFileRejection, FileAttributes] = + override def apply(description: FileDescription, entity: BodyPartEntity): IO[SaveFileRejection, FileAttributes] = initLocation(description.uuid, description.filename).flatMap { case (fullPath, relativePath) => IO.deferFuture( - source.runWith(SinkUtils.combineMat(digestSink(storage.value.algorithm), FileIO.toPath(fullPath, openOpts)) { - case (digest, ioResult) if fullPath.toFile.exists() => - Future.successful( - FileAttributes( - uuid = description.uuid, - location = Uri(fullPath.toUri.toString), - path = relativePath, - filename = description.filename, - mediaType = description.mediaType, - bytes = ioResult.count, - digest = digest, - origin = Client + entity.dataBytes.runWith( + SinkUtils.combineMat(digestSink(storage.value.algorithm), FileIO.toPath(fullPath, openOpts)) { + case (digest, ioResult) if fullPath.toFile.exists() => + Future.successful( + FileAttributes( + uuid = description.uuid, + location = Uri(fullPath.toUri.toString), + path = relativePath, + filename = description.filename, + mediaType = description.mediaType, + bytes = ioResult.count, + digest = digest, + origin = Client + ) ) - ) - case _ => - Future.failed(new IllegalArgumentException("File was not written")) - }) + case _ => + Future.failed(new IllegalArgumentException("File was not written")) + } + ) ).mapError { case _: FileAlreadyExistsException => ResourceAlreadyExists(fullPath.toString) case err => UnexpectedSaveError(fullPath.toString, err.getMessage) diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/RemoteDiskStorageSaveFile.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/RemoteDiskStorageSaveFile.scala index 3e7d602540..f55f1463c8 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/RemoteDiskStorageSaveFile.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/RemoteDiskStorageSaveFile.scala @@ -1,6 +1,7 @@ package ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote import akka.actor.ActorSystem +import akka.http.scaladsl.model.BodyPartEntity import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileAttributes.FileAttributesOrigin.Client import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{FileAttributes, FileDescription} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.StoragesConfig.StorageTypeConfig @@ -10,7 +11,6 @@ import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.SaveFil import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.StorageFileRejection.SaveFileRejection import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.client.RemoteDiskStorageClient import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.client.model.RemoteDiskStorageFileAttributes -import ch.epfl.bluebrain.nexus.delta.sdk.AkkaSource import ch.epfl.bluebrain.nexus.delta.sdk.http.HttpClient import ch.epfl.bluebrain.nexus.delta.sdk.model.identities.AuthToken import monix.bio.IO @@ -25,10 +25,10 @@ class RemoteDiskStorageSaveFile(storage: RemoteDiskStorage)(implicit override def apply( description: FileDescription, - source: AkkaSource + entity: BodyPartEntity ): IO[SaveFileRejection, FileAttributes] = { val path = intermediateFolders(storage.project, description.uuid, description.filename) - client.createFile(storage.value.folder, path, source).map { + client.createFile(storage.value.folder, path, entity).map { case RemoteDiskStorageFileAttributes(location, bytes, digest, mediaType) => FileAttributes( uuid = description.uuid, diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/client/RemoteDiskStorageClient.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/client/RemoteDiskStorageClient.scala index 0b2aa7ffc6..23cbaacc86 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/client/RemoteDiskStorageClient.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/client/RemoteDiskStorageClient.scala @@ -2,8 +2,7 @@ package ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote import akka.actor.ActorSystem import akka.http.scaladsl.client.RequestBuilding._ -import akka.http.scaladsl.model.ContentTypes.`application/octet-stream` -import akka.http.scaladsl.model.HttpEntity +import akka.http.scaladsl.model.BodyPartEntity import akka.http.scaladsl.model.Multipart.FormData import akka.http.scaladsl.model.Multipart.FormData.BodyPart import akka.http.scaladsl.model.StatusCodes._ @@ -26,6 +25,7 @@ import io.circe.generic.semiauto.deriveDecoder import io.circe.syntax._ import io.circe.{Decoder, Json} import monix.bio.{IO, UIO} + import scala.concurrent.duration._ /** @@ -72,13 +72,12 @@ final class RemoteDiskStorageClient(baseUri: BaseUri)(implicit client: HttpClien * @param source * the file content */ - def createFile(bucket: Label, relativePath: Path, source: AkkaSource)(implicit + def createFile(bucket: Label, relativePath: Path, entity: BodyPartEntity)(implicit cred: Option[AuthToken] ): IO[SaveFileRejection, RemoteDiskStorageFileAttributes] = { - val endpoint = baseUri.endpoint / "buckets" / bucket.value / "files" / relativePath - val bodyPartEntity = HttpEntity.IndefiniteLength(`application/octet-stream`, source) - val filename = relativePath.lastSegment.getOrElse("filename") - val multipartForm = FormData(BodyPart("file", bodyPartEntity, Map("filename" -> filename))).toEntity() + val endpoint = baseUri.endpoint / "buckets" / bucket.value / "files" / relativePath + val filename = relativePath.lastSegment.getOrElse("filename") + val multipartForm = FormData(BodyPart("file", entity, Map("filename" -> filename))).toEntity() client.fromJsonTo[RemoteDiskStorageFileAttributes](Put(endpoint, multipartForm).withCredentials).mapError { case HttpClientStatusError(_, `Conflict`, _) => SaveFileRejection.ResourceAlreadyExists(relativePath.toString) diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/s3/S3StorageSaveFile.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/s3/S3StorageSaveFile.scala index 7b7fc6320a..b0fa7c391e 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/s3/S3StorageSaveFile.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/s3/S3StorageSaveFile.scala @@ -1,15 +1,14 @@ package ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.s3 import akka.actor.ActorSystem -import akka.http.scaladsl.model.Uri import akka.http.scaladsl.model.Uri.Path.Slash -import akka.stream.alpakka.s3.{S3Attributes, S3Exception} +import akka.http.scaladsl.model.{BodyPartEntity, Uri} import akka.stream.alpakka.s3.scaladsl.S3 +import akka.stream.alpakka.s3.{S3Attributes, S3Exception} import akka.stream.scaladsl.Sink import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileAttributes.FileAttributesOrigin.Client import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{FileAttributes, FileDescription} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.StoragesConfig.StorageTypeConfig -import ch.epfl.bluebrain.nexus.delta.sdk.AkkaSource import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.Storage.S3Storage import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.SaveFile import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.SaveFile.{digestSink, intermediateFolders, sizeSink} @@ -27,7 +26,7 @@ final class S3StorageSaveFile(storage: S3Storage)(implicit config: StorageTypeCo override def apply( description: FileDescription, - source: AkkaSource + entity: BodyPartEntity ): IO[SaveFileRejection, FileAttributes] = { val attributes = S3Attributes.settings(storage.value.alpakkaSettings(config)) val path = intermediateFolders(storage.project, description.uuid, description.filename) @@ -39,7 +38,7 @@ final class S3StorageSaveFile(storage: S3Storage)(implicit config: StorageTypeCo .runWith(Sink.last) .flatMap { case None => - source.runWith(SinkUtils.combineMat(digestSink(storage.value.algorithm), sizeSink, s3Sink) { + entity.dataBytes.runWith(SinkUtils.combineMat(digestSink(storage.value.algorithm), sizeSink, s3Sink) { case (digest, bytes, s3Result) => Future.successful( FileAttributes( diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FormDataExtractorSpec.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FormDataExtractorSpec.scala index 94468aef9e..563d45f2c1 100644 --- a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FormDataExtractorSpec.scala +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FormDataExtractorSpec.scala @@ -42,10 +42,10 @@ class FormDataExtractorSpec ) .toEntity() - val expectedDescription = FileDescription(uuid, "file.txt", Some(`text/plain(UTF-8)`)) - val (description, source) = extractor(iri, entity, 179, None).accepted + val expectedDescription = FileDescription(uuid, "file.txt", Some(`text/plain(UTF-8)`)) + val (description, resultEntity) = extractor(iri, entity, 179, None).accepted description shouldEqual expectedDescription - consume(source) shouldEqual content + consume(resultEntity.dataBytes) shouldEqual content } "fail to be extracted if no file part exists found" in { diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskStorageSaveFileSpec.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskStorageSaveFileSpec.scala index 32ab9b2c72..6709f17b8f 100644 --- a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskStorageSaveFileSpec.scala +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskStorageSaveFileSpec.scala @@ -2,10 +2,8 @@ package ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.disk import akka.actor.ActorSystem import akka.http.scaladsl.model.ContentTypes.`text/plain(UTF-8)` -import akka.http.scaladsl.model.Uri -import akka.stream.scaladsl.Source +import akka.http.scaladsl.model.{HttpEntity, Uri} import akka.testkit.TestKit -import akka.util.ByteString import ch.epfl.bluebrain.nexus.delta.kernel.Secret import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.Digest.ComputedDigest import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileAttributes.FileAttributesOrigin.Client @@ -51,12 +49,12 @@ class DiskStorageSaveFileSpec val storage = DiskStorage(iri, project, value, Map.empty, Secret(Json.obj())) val uuid = UUID.fromString("8049ba90-7cc6-4de5-93a1-802c04200dcc") val content = "file content" - val source = Source(content.map(c => ByteString(c.toString))) + val entity = HttpEntity(content) "save a file to a volume" in { val description = FileDescription(uuid, "myfile.txt", Some(`text/plain(UTF-8)`)) - val attributes = storage.saveFile.apply(description, source).accepted + val attributes = storage.saveFile.apply(description, entity).accepted Files.readString(file.value) shouldEqual content @@ -78,7 +76,7 @@ class DiskStorageSaveFileSpec "fail attempting to save the same file again" in { val description = FileDescription(uuid, "myfile.txt", Some(`text/plain(UTF-8)`)) - storage.saveFile.apply(description, source).rejectedWith[ResourceAlreadyExists] + storage.saveFile.apply(description, entity).rejectedWith[ResourceAlreadyExists] } } diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/RemoteStorageSaveAndFetchFileSpec.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/RemoteStorageSaveAndFetchFileSpec.scala index 74761a1426..7c9dd3a52d 100644 --- a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/RemoteStorageSaveAndFetchFileSpec.scala +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/RemoteStorageSaveAndFetchFileSpec.scala @@ -2,10 +2,8 @@ package ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote import akka.actor.ActorSystem import akka.http.scaladsl.model.ContentTypes.`text/plain(UTF-8)` -import akka.http.scaladsl.model.Uri -import akka.stream.scaladsl.Source +import akka.http.scaladsl.model.{HttpEntity, Uri} import akka.testkit.TestKit -import akka.util.ByteString import ch.epfl.bluebrain.nexus.delta.kernel.Secret import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.Digest.ComputedDigest import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileAttributes.FileAttributesOrigin.Client @@ -77,7 +75,7 @@ class RemoteStorageSaveAndFetchFileSpec(docker: RemoteStorageDocker) "RemoteDiskStorage operations" should { val content = "file content" - val source = Source(content.map(c => ByteString(c.toString))) + val entity = HttpEntity(content) val attributes = FileAttributes( uuid, @@ -92,7 +90,7 @@ class RemoteStorageSaveAndFetchFileSpec(docker: RemoteStorageDocker) "save a file to a folder" in { val description = FileDescription(uuid, filename, Some(`text/plain(UTF-8)`)) - storage.saveFile.apply(description, source).accepted shouldEqual attributes + storage.saveFile.apply(description, entity).accepted shouldEqual attributes } "fetch a file from a folder" in { @@ -113,7 +111,7 @@ class RemoteStorageSaveAndFetchFileSpec(docker: RemoteStorageDocker) "fail attempting to save the same file again" in { val description = FileDescription(uuid, "myfile.txt", Some(`text/plain(UTF-8)`)) - storage.saveFile.apply(description, source).rejectedWith[ResourceAlreadyExists] + storage.saveFile.apply(description, entity).rejectedWith[ResourceAlreadyExists] } } } diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/client/RemoteStorageClientSpec.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/client/RemoteStorageClientSpec.scala index 05c05b3079..e57f5ce900 100644 --- a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/client/RemoteStorageClientSpec.scala +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/client/RemoteStorageClientSpec.scala @@ -2,10 +2,8 @@ package ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote import akka.actor.ActorSystem import akka.http.scaladsl.model.ContentTypes.`text/plain(UTF-8)` -import akka.http.scaladsl.model.{StatusCodes, Uri} -import akka.stream.scaladsl.Source +import akka.http.scaladsl.model.{HttpEntity, StatusCodes, Uri} import akka.testkit.TestKit -import akka.util.ByteString import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.Digest.{ComputedDigest, NotComputedDigest} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.DigestAlgorithm import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.AkkaSourceHelpers @@ -56,7 +54,7 @@ class RemoteStorageClientSpec(docker: RemoteStorageDocker) implicit val cred: Option[AuthToken] = None val content = RemoteStorageDocker.Content - val source = Source(content.map(c => ByteString(c.toString))) + val entity = HttpEntity(content) val attributes = RemoteDiskStorageFileAttributes( location = s"file:///app/$BucketName/nexus/my/file.txt", bytes = 12, @@ -75,7 +73,7 @@ class RemoteStorageClientSpec(docker: RemoteStorageDocker) } "create a file" in { - client.createFile(bucket, Uri.Path("my/file.txt"), source).accepted shouldEqual attributes + client.createFile(bucket, Uri.Path("my/file.txt"), entity).accepted shouldEqual attributes } "get a file" in { diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/s3/S3StorageLinkFileSpec.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/s3/S3StorageLinkFileSpec.scala index 134aca366d..8e20393ac5 100644 --- a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/s3/S3StorageLinkFileSpec.scala +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/s3/S3StorageLinkFileSpec.scala @@ -2,10 +2,8 @@ package ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.s3 import akka.actor.ActorSystem import akka.http.scaladsl.model.ContentTypes.`text/plain(UTF-8)` -import akka.http.scaladsl.model.Uri -import akka.stream.scaladsl.Source +import akka.http.scaladsl.model.{HttpEntity, Uri} import akka.testkit.TestKit -import akka.util.ByteString import ch.epfl.bluebrain.nexus.delta.kernel.Secret import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.Digest.ComputedDigest import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileAttributes.FileAttributesOrigin @@ -88,12 +86,12 @@ class S3StorageLinkFileSpec(docker: MinioDocker) "S3Storage linking operations" should { val content = "file content" - val source = Source(content.map(c => ByteString(c.toString))) + val entity = HttpEntity(content) val description = FileDescription(uuid, filename, Some(`text/plain(UTF-8)`)) "succeed" in { - storage.saveFile.apply(description, source).accepted shouldEqual attributes + storage.saveFile.apply(description, entity).accepted shouldEqual attributes val linkAttributes = attributes.copy(origin = FileAttributesOrigin.Storage) storage.linkFile.apply(attributes.path, description).accepted shouldEqual linkAttributes diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/s3/S3StorageSaveAndFetchFileSpec.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/s3/S3StorageSaveAndFetchFileSpec.scala index b43e33084f..241307c9d6 100644 --- a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/s3/S3StorageSaveAndFetchFileSpec.scala +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/s3/S3StorageSaveAndFetchFileSpec.scala @@ -2,23 +2,20 @@ package ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.s3 import akka.actor.ActorSystem import akka.http.scaladsl.model.ContentTypes.`text/plain(UTF-8)` -import akka.http.scaladsl.model.Uri -import akka.stream.scaladsl.Source +import akka.http.scaladsl.model.{HttpEntity, Uri} import akka.testkit.TestKit -import akka.util.ByteString -import cats.implicits._ import ch.epfl.bluebrain.nexus.delta.kernel.Secret import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.Digest.ComputedDigest import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileAttributes.FileAttributesOrigin.Client import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{FileAttributes, FileDescription} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.StorageFixtures +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.DigestAlgorithm import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.Storage.S3Storage import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.StorageValue.S3StorageValue -import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.DigestAlgorithm +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.AkkaSourceHelpers import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.StorageFileRejection.FetchFileRejection.{FileNotFound, UnexpectedFetchError} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.StorageFileRejection.SaveFileRejection.{ResourceAlreadyExists, UnexpectedSaveError} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.s3.MinioSpec._ -import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.AkkaSourceHelpers import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.permissions.{read, write} import ch.epfl.bluebrain.nexus.delta.sdk.model.projects.ProjectRef import ch.epfl.bluebrain.nexus.delta.sdk.syntax._ @@ -27,9 +24,9 @@ import ch.epfl.bluebrain.nexus.testkit.minio.MinioDocker import ch.epfl.bluebrain.nexus.testkit.minio.MinioDocker._ import io.circe.Json import monix.execution.Scheduler -import org.scalatest.{BeforeAndAfterAll, DoNotDiscover} import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpecLike +import org.scalatest.{BeforeAndAfterAll, DoNotDiscover} import software.amazon.awssdk.regions.Region import java.util.UUID @@ -92,17 +89,17 @@ class S3StorageSaveAndFetchFileSpec(docker: MinioDocker) "S3Storage operations" should { val content = "file content" - val source = Source(content.map(c => ByteString(c.toString))) + val entity = HttpEntity(content) "fail saving a file to a bucket on wrong credentials" in { val description = FileDescription(uuid, filename, Some(`text/plain(UTF-8)`)) val otherStorage = storage.copy(value = storage.value.copy(accessKey = Some(Secret("wrong")))) - otherStorage.saveFile.apply(description, source).rejectedWith[UnexpectedSaveError] + otherStorage.saveFile.apply(description, entity).rejectedWith[UnexpectedSaveError] } "save a file to a bucket" in { val description = FileDescription(uuid, filename, Some(`text/plain(UTF-8)`)) - storage.saveFile.apply(description, source).accepted shouldEqual attributes + storage.saveFile.apply(description, entity).accepted shouldEqual attributes } "fetch a file from a bucket" in { @@ -121,7 +118,7 @@ class S3StorageSaveAndFetchFileSpec(docker: MinioDocker) "fail attempting to save the same file again" in { val description = FileDescription(uuid, "myfile.txt", Some(`text/plain(UTF-8)`)) - storage.saveFile.apply(description, source).rejectedWith[ResourceAlreadyExists] + storage.saveFile.apply(description, entity).rejectedWith[ResourceAlreadyExists] } } } diff --git a/tests/src/test/resources/kg/files/empty b/tests/src/test/resources/kg/files/empty new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/src/test/resources/kg/storages/statistics.json b/tests/src/test/resources/kg/storages/statistics.json index dd62ecb681..ecc1049f3d 100644 --- a/tests/src/test/resources/kg/storages/statistics.json +++ b/tests/src/test/resources/kg/storages/statistics.json @@ -1,5 +1,5 @@ { "@context": "https://bluebrain.github.io/nexus/contexts/storages.json", - "files": 3, + "files": 4, "spaceUsed": 125 } \ No newline at end of file diff --git a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/StorageSpec.scala b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/StorageSpec.scala index c4456758e9..2d2120ef27 100644 --- a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/StorageSpec.scala +++ b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/StorageSpec.scala @@ -78,6 +78,29 @@ abstract class StorageSpec extends BaseSpec with CirceEq { s"uploading an attachment against the $storageName storage" should { + "upload empty file" in { + deltaClient.putAttachment[Json]( + s"/files/$fullId/empty?storage=nxv:$storageId", + contentOf("/kg/files/empty"), + ContentTypes.`text/plain(UTF-8)`, + "empty", + Coyote + ) { (_, response) => + response.status shouldEqual StatusCodes.Created + } + } + + "fetch empty file" in { + deltaClient.get[ByteString](s"/files/$fullId/attachment:empty", Coyote, acceptAll) { (content, response) => + assertFetchAttachment( + response, + "empty", + ContentTypes.`text/plain(UTF-8)` + ) + content.utf8String shouldEqual contentOf("/kg/files/empty") + } + } + "upload attachment with JSON" in { deltaClient.putAttachment[Json]( s"/files/$fullId/attachment.json?storage=nxv:$storageId",