Storage prefix as path (#5068)

delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/StoragePluginModule.scala

@@ -1,7 +1,7 @@
 package ch.epfl.bluebrain.nexus.delta.plugins.storage
 
 import akka.actor.typed.ActorSystem
-import akka.http.scaladsl.model.Uri
+import akka.http.scaladsl.model.Uri.Path
 import cats.effect.{Clock, IO}
 import ch.epfl.bluebrain.nexus.delta.kernel.utils.{ClasspathResourceLoader, TransactionalFileCopier, UUIDF}
 import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.client.ElasticSearchClient
@@ -74,7 +74,7 @@ class StoragePluginModule(priority: Int) extends ModuleDef {
  }
 
  make[S3LocationGenerator].from { (cfg: StoragePluginConfig) =>
- val prefix: Uri = cfg.storages.storageTypeConfig.amazon.flatMap(_.prefix).getOrElse(Uri.Empty)
+ val prefix: Path = cfg.storages.storageTypeConfig.amazon.flatMap(_.prefix).getOrElse(Path.Empty)
  new S3LocationGenerator(prefix)
  }
 

delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/StoragesConfig.scala

@@ -1,6 +1,7 @@
 package ch.epfl.bluebrain.nexus.delta.plugins.storage.storages
 
 import akka.http.scaladsl.model.Uri
+import akka.http.scaladsl.model.Uri.Path
 import cats.implicits._
 import ch.epfl.bluebrain.nexus.delta.kernel.Secret
 import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.StoragesConfig.StorageTypeConfig
@@ -181,9 +182,9 @@ object StoragesConfig {
  showLocation: Boolean,
  defaultMaxFileSize: Long,
  defaultBucket: String,
- prefix: Option[Uri]
+ prefix: Option[Path]
  ) extends StorageTypeEntryConfig {
- val prefixUri: Uri = prefix.getOrElse(Uri.Empty)
+ val prefixPath: Path = prefix.getOrElse(Path.Empty)
  }
 
  /**
@@ -218,13 +219,16 @@ object StoragesConfig {
  implicit private val uriConverter: ConfigConvert[Uri] =
  ConfigConvert.viaString[Uri](catchReadError(Uri(_)), _.toString)
 
+ implicit private val pathConverter: ConfigConvert[Path] =
+ ConfigConvert.viaString[Path](catchReadError(Path(_)), _.toString)
+
  implicit private val permissionConverter: ConfigConvert[Permission] =
  ConfigConvert.viaString[Permission](optF(Permission(_).toOption), _.toString)
 
  implicit private val digestAlgConverter: ConfigConvert[DigestAlgorithm] =
  ConfigConvert.viaString[DigestAlgorithm](optF(DigestAlgorithm(_)), _.toString)
 
- implicit private val pathConverter: ConfigConvert[AbsolutePath] =
+ implicit private val absolutePathConverter: ConfigConvert[AbsolutePath] =
  ConfigConvert.viaString[AbsolutePath](
  str => AbsolutePath(str).leftMap(err => CannotConvert(str, "AbsolutePath", err)),
  _.toString

delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/s3/S3LocationGenerator.scala

@@ -1,17 +1,20 @@
 package ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.s3
 
 import akka.http.scaladsl.model.Uri
+import akka.http.scaladsl.model.Uri.Path
 import ch.epfl.bluebrain.nexus.delta.rdf.syntax.uriSyntax
 import ch.epfl.bluebrain.nexus.delta.sourcing.model.ProjectRef
 
 import java.util.UUID
 
-class S3LocationGenerator(prefix: Uri) {
+class S3LocationGenerator(prefix: Path) {
 
  private def uuidToPath(uuid: UUID) =
  Uri.Path(uuid.toString.toLowerCase.takeWhile(_ != '-').mkString("/"))
 
  def file(project: ProjectRef, uuid: UUID, filename: String): Uri =
- prefix / project.organization.toString / project.project.toString / "files" / uuidToPath(uuid) / filename
+ Uri.Empty / prefix / project.organization.toString / project.project.toString / "files" / uuidToPath(
+ uuid
+ ) / filename
 
 }

delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/mocks/FileOperationsMock.scala

@@ -2,6 +2,7 @@ package ch.epfl.bluebrain.nexus.delta.plugins.storage.files.mocks
 
 import akka.actor.ActorSystem
 import akka.http.scaladsl.model.Uri
+import akka.http.scaladsl.model.Uri.Path
 import cats.effect.IO
 import ch.epfl.bluebrain.nexus.delta.kernel.utils.UUIDF
 import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.UploadedFileInformation
@@ -30,7 +31,7 @@ object FileOperationsMock {
  FileOperations.mk(
  DiskFileOperations.mk,
  RemoteDiskFileOperations.mk(RemoteDiskStorageClientDisabled),
- S3FileOperations.mk(S3StorageClient.disabled, new S3LocationGenerator(Uri.Empty))
+ S3FileOperations.mk(S3StorageClient.disabled, new S3LocationGenerator(Path.Empty))
  )
 
  def unimplemented: FileOperations = new FileOperations {

delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/s3/LocalStackS3StorageClient.scala

@@ -53,7 +53,7 @@ object LocalStackS3StorageClient {
  showLocation = false,
  defaultMaxFileSize = 1,
  defaultBucket = defaultBucket,
- prefix = Some(prefix)
+ prefix = Some(Uri.Path(prefix))
  )
  (S3StorageClient.unsafe(client), client, conf)
  }

delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/s3/S3FileOperationsSuite.scala

@@ -43,7 +43,7 @@ class S3FileOperationsSuite
  localStackS3Client()
  implicit private lazy val as: ActorSystem = actorSystem()
 
- private lazy val locationGenerator = new S3LocationGenerator(conf.prefixUri)
+ private lazy val locationGenerator = new S3LocationGenerator(conf.prefixPath)
  private lazy val fileOps = S3FileOperations.mk(s3StorageClient, locationGenerator)
 
  private def makeDigest(content: String): ComputedDigest = {
@@ -54,7 +54,8 @@ class S3FileOperationsSuite
  private def expectedPath(proj: ProjectRef, filename: String): Uri.Path =
  Uri.Path(s"$proj/files/${randomUuid.toString.takeWhile(_ != '-').mkString("/")}/$filename")
 
- private def expectedLocation(proj: ProjectRef, filename: String): Uri = conf.prefixUri / expectedPath(proj, filename)
+ private def expectedLocation(proj: ProjectRef, filename: String): Uri =
+ Uri.Empty / conf.prefixPath / expectedPath(proj, filename)
 
  test("List objects in an existing bucket") {
  givenAnS3Bucket { bucket =>

delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/s3/S3LocationGeneratorSuite.scala

@@ -1,6 +1,7 @@
 package ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.s3
 
 import akka.http.scaladsl.model.Uri
+import akka.http.scaladsl.model.Uri.Path
 import ch.epfl.bluebrain.nexus.delta.sourcing.model.ProjectRef
 import ch.epfl.bluebrain.nexus.testkit.mu.NexusSuite
 
@@ -9,7 +10,8 @@ import java.util.UUID
 class S3LocationGeneratorSuite extends NexusSuite {
 
  test("Generate the expected uri") {
- val prefix = Uri("/prefix")
+ val prefix = Path("/prefix")
+ println(prefix)
  val project = ProjectRef.unsafe("org", "project")
  val uuid = UUID.fromString("12345678-b2e3-40b9-93de-c809415d7640")
  val filename = "cat.gif"

delta/rdf/src/main/scala/ch/epfl/bluebrain/nexus/delta/rdf/instances/UriInstances.scala

@@ -30,6 +30,11 @@ trait UriInstances {
  .toEither
  .leftMap(err => CannotConvert(str, classOf[Uri].getSimpleName, err.getMessage))
  )
+
+ implicit val pathConfigReader: ConfigReader[Uri.Path] = ConfigReader.fromString(str =>
+ Try(Uri.Path(str)).toEither
+ .leftMap(err => CannotConvert(str, classOf[Uri.Path].getSimpleName, err.getMessage))
+ )
 }
 
 object UriInstances extends UriInstances

ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/config/FileProcessingConfig.scala

@@ -1,14 +1,14 @@
 package ch.epfl.bluebrain.nexus.ship.config
 
-import akka.http.scaladsl.model.Uri
+import akka.http.scaladsl.model.Uri.Path
 import pureconfig.ConfigReader
 import ch.epfl.bluebrain.nexus.delta.sdk.instances._
 import pureconfig.generic.semiauto.deriveReader
 
 final case class FileProcessingConfig(
  importBucket: String,
  targetBucket: String,
- prefix: Option[Uri],
+ prefix: Option[Path],
  skipFileEvents: Boolean
 )
 

ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/files/FileCopier.scala

@@ -1,6 +1,7 @@
 package ch.epfl.bluebrain.nexus.ship.files
 
 import akka.http.scaladsl.model.Uri
+import akka.http.scaladsl.model.Uri.Path
 import cats.effect.IO
 import ch.epfl.bluebrain.nexus.delta.kernel.Logger
 import ch.epfl.bluebrain.nexus.delta.kernel.utils.UrlUtils
@@ -42,7 +43,7 @@ object FileCopier {
  ): FileCopier = {
  val importBucket = config.importBucket
  val targetBucket = config.targetBucket
- val locationGenerator = new S3LocationGenerator(config.prefix.getOrElse(Uri.Empty))
+ val locationGenerator = new S3LocationGenerator(config.prefix.getOrElse(Path.Empty))
  (project: ProjectRef, attributes: FileAttributes) => {
  val origin = attributes.path
  val patchedFileName = if (attributes.filename.isEmpty) "file" else attributes.filename

ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/config/ShipConfigFixtures.scala

@@ -1,10 +1,11 @@
 package ch.epfl.bluebrain.nexus.ship.config
 
+import akka.http.scaladsl.model.Uri
 import ch.epfl.bluebrain.nexus.delta.kernel.Secret
 import ch.epfl.bluebrain.nexus.delta.plugins.storage.files
 import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.StoragesConfig.S3StorageConfig
 import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.{permissions, StorageFixtures, StoragesConfig}
-import ch.epfl.bluebrain.nexus.delta.rdf.syntax.{iriStringContextSyntax, uriStringContextSyntax}
+import ch.epfl.bluebrain.nexus.delta.rdf.syntax.iriStringContextSyntax
 import ch.epfl.bluebrain.nexus.delta.sdk.identities.model.ServiceAccount
 import ch.epfl.bluebrain.nexus.delta.sdk.model.{BaseUri, ServiceAccountConfig}
 import ch.epfl.bluebrain.nexus.delta.sdk.{ConfigFixtures, Defaults}
@@ -59,7 +60,7 @@ trait ShipConfigFixtures extends ConfigFixtures with StorageFixtures with Classp
  FileProcessingConfig(
  importBucket,
  targetBucket,
- Some(uri"/prefix"),
+ Some(Uri.Path("/prefix")),
  skipFileEvents = false
  ),
  IriPatcherConfig(enabled = false, iri"https://bbp.epfl.ch/", iri"https:/openbrainplatform.com/"),

tests/docker/config/delta-postgres.conf

@@ -156,7 +156,7 @@ plugins {
  default-access-key = "MY_ACCESS_KEY"
  default-secret-key = "CHUTCHUT"
  default-bucket = "mydefaultbucket"
- prefix = "myprefix"
+ prefix = "/myprefix"
  }
  }
 

tests/src/test/resources/application.conf

@@ -21,6 +21,6 @@ storage {
  s-3 {
  access-key = "MY_ACCESS_KEY"
  secret-key = "CHUTCHUT"
- prefix = "myprefix"
+ prefix = "/myprefix"
  }
 }

tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/files/S3StorageSpec.scala

@@ -231,7 +231,7 @@ class S3StorageSpec extends StorageSpec {
 
  private def assertThereIsAFileInS3WithAtLocation(location: String): Assertion = {
  s3Client
- .listObjectsV2(ListObjectsV2Request.builder.bucket(bucket).prefix(s"myprefix/$projectRef/files").build)
+ .listObjectsV2(ListObjectsV2Request.builder.bucket(bucket).prefix(s"/myprefix/$projectRef/files").build)
  .map(_.contents.asScala.map(_.key()))
  .map(keys => keys should contain(location))
  .accepted