Introduce etag header in dowloading file operation (#5165)

* Introduce etag header in dowloading file operation

---------

Co-authored-by: Simon Dumas <[email protected]>

delta/app/src/main/scala/ch/epfl/bluebrain/nexus/delta/routes/SchemaJobRoutes.scala

@@ -55,7 +55,7 @@ class SchemaJobRoutes(
  )
  }
  }.map { s =>
- FileResponse("validation.json", ContentTypes.`application/json`, None, s)
+ FileResponse("validation.json", ContentTypes.`application/json`, None, None, None, s)
  }
 
  def routes: Route =

delta/plugins/archive/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/archive/routes/ArchiveRoutes.scala

@@ -77,9 +77,7 @@ class ArchiveRoutes(
  emit(statusCode, io.mapValue(_.metadata).attemptNarrow[ArchiveRejection])
 
  private def emitArchiveFile(source: IO[AkkaSource]) = {
- val response = source.map { s =>
- FileResponse(s"archive.zip", Zip.contentType, None, s)
- }
+ val response = source.map { s => FileResponse.noCache(s"archive.zip", Zip.contentType, None, s) }
  emit(response.attemptNarrow[ArchiveRejection])
  }
 

delta/plugins/archive/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/archive/ArchiveDownloadSpec.scala

@@ -2,25 +2,25 @@ package ch.epfl.bluebrain.nexus.delta.plugins.archive
 
 import akka.actor.ActorSystem
 import akka.http.scaladsl.model.ContentTypes.`text/plain(UTF-8)`
-import akka.http.scaladsl.model.{ContentTypes, Uri}
+import akka.http.scaladsl.model.Uri
 import akka.stream.scaladsl.Source
 import akka.testkit.TestKit
 import akka.util.ByteString
 import cats.data.NonEmptySet
 import cats.effect.IO
 import ch.epfl.bluebrain.nexus.delta.kernel.utils.UrlUtils.encode
-import ch.epfl.bluebrain.nexus.delta.plugins.storage.FileSelf.ParsingError
 import ch.epfl.bluebrain.nexus.delta.plugins.archive.model.ArchiveReference.{FileReference, FileSelfReference, ResourceReference}
 import ch.epfl.bluebrain.nexus.delta.plugins.archive.model.ArchiveRejection.{InvalidFileSelf, ResourceNotFound}
 import ch.epfl.bluebrain.nexus.delta.plugins.archive.model.{ArchiveRejection, ArchiveValue}
-import ch.epfl.bluebrain.nexus.delta.plugins.storage.{FileSelf, RemoteContextResolutionFixture}
+import ch.epfl.bluebrain.nexus.delta.plugins.storage.FileSelf.ParsingError
 import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.generators.FileGen
 import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileAttributes.FileAttributesOrigin.Client
 import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileRejection.FileNotFound
 import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{Digest, FileAttributes}
 import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.schemas
 import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.StorageFixtures
 import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.AbsolutePath
+import ch.epfl.bluebrain.nexus.delta.plugins.storage.{FileSelf, RemoteContextResolutionFixture}
 import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri
 import ch.epfl.bluebrain.nexus.delta.rdf.Vocabulary.nxv
 import ch.epfl.bluebrain.nexus.delta.rdf.utils.JsonKeyOrdering
@@ -123,21 +123,11 @@ class ArchiveDownloadSpec
  val fetchFileContent: (Iri, ProjectRef) => IO[FileResponse] = {
  case (`id1`, `projectRef`) =>
  IO.pure(
- FileResponse(
- file1Name,
- ContentTypes.`text/plain(UTF-8)`,
- Some(file1Size),
- Source.single(ByteString(file1Content))
- )
+ FileResponse.noCache(file1Name, `text/plain(UTF-8)`, Some(file1Size), Source.single(ByteString(file1Content)))
  )
  case (`id2`, `projectRef`) =>
  IO.pure(
- FileResponse(
- file2Name,
- ContentTypes.`text/plain(UTF-8)`,
- Some(file2Size),
- Source.single(ByteString(file2Content))
- )
+ FileResponse.noCache(file2Name, `text/plain(UTF-8)`, Some(file2Size), Source.single(ByteString(file2Content)))
  )
  case (id, ref) =>
  IO.raiseError(FileNotFound(id, ref))

delta/plugins/archive/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/archive/ArchiveRoutesSpec.scala

@@ -4,15 +4,15 @@ import akka.http.scaladsl.model.ContentTypes.`text/plain(UTF-8)`
 import akka.http.scaladsl.model.MediaRanges.`*/*`
 import akka.http.scaladsl.model.MediaTypes.`application/zip`
 import akka.http.scaladsl.model.headers.{`Content-Type`, Accept, Location, OAuth2BearerToken}
-import akka.http.scaladsl.model.{ContentTypes, StatusCodes, Uri}
+import akka.http.scaladsl.model.{StatusCodes, Uri}
 import akka.http.scaladsl.server.Route
 import akka.stream.scaladsl.Source
 import akka.util.ByteString
 import cats.effect.IO
 import ch.epfl.bluebrain.nexus.delta.kernel.utils.UrlUtils.encode
 import ch.epfl.bluebrain.nexus.delta.kernel.utils.{StatefulUUIDF, UUIDF}
-import ch.epfl.bluebrain.nexus.delta.plugins.storage.FileSelf.ParsingError.InvalidPath
 import ch.epfl.bluebrain.nexus.delta.plugins.archive.routes.ArchiveRoutes
+import ch.epfl.bluebrain.nexus.delta.plugins.storage.FileSelf.ParsingError.InvalidPath
 import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.generators.FileGen
 import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.Digest.ComputedDigest
 import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileAttributes.FileAttributesOrigin.Client
@@ -129,7 +129,7 @@ class ArchiveRoutesSpec extends BaseRouteSpec with StorageFixtures with ArchiveH
  IO.raiseError(AuthorizationFailed(AclAddress.Project(p), Permission.unsafe("disk/read")))
  case (`fileId`, `projectRef`, _) =>
  IO.pure(
- FileResponse("file.txt", ContentTypes.`text/plain(UTF-8)`, Some(12L), Source.single(ByteString(fileContent)))
+ FileResponse.noCache("file.txt", `text/plain(UTF-8)`, Some(12L), Source.single(ByteString(fileContent)))
  )
  case (id, ref, _) =>
  IO.raiseError(FileNotFound(id, ref))

delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/Files.scala

@@ -465,7 +465,14 @@ final class Files(
  _ <- validateAuth(id.project, storage.value.storageValue.readPermission)
  s = fetchFile(storage.value, attributes, file.id)
  mediaType = attributes.mediaType.getOrElse(`application/octet-stream`)
- } yield FileResponse(attributes.filename, mediaType, Some(attributes.bytes), s.attemptNarrow[FileRejection])
+ } yield FileResponse(
+ attributes.filename,
+ mediaType,
+ Some(ResourceF.etagValue(file)),
+ Some(file.updatedAt),
+ Some(attributes.bytes),
+ s.attemptNarrow[FileRejection]
+ )
  }.span("fetchFileContent")
 
  private def fetchFile(storage: Storage, attr: FileAttributes, fileId: Iri): IO[AkkaSource] =

delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/FilesRoutesSpec.scala

@@ -658,6 +658,9 @@ class FilesRoutesSpec
  header("Content-Disposition").value.value() shouldEqual
  s"""attachment; filename="=?UTF-8?B?${base64encode(id)}?=""""
  response.asString shouldEqual content
+ val attr = attributes(id)
+ response.header[`Content-Length`].value shouldEqual `Content-Length`(attr.bytes)
+ response.expectConditionalCacheHeaders
  response.headers should contain(varyHeader)
  }
  }

delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/directives/DeltaDirectives.scala

@@ -116,6 +116,13 @@ trait DeltaDirectives extends UriDirectives {
  }
  }
 
+ /**
+ * Returns the best of the given encoding alternatives given the preferences the client indicated in the request's
+ * `Accept-Encoding` headers.
+ *
+ * This implementation is based on the akka internal implemetation in
+ * `akka.http.scaladsl.server.directives.CodingDirectives#_encodeResponse`
+ */
  def requestEncoding: Directive1[HttpEncoding] =
  extractRequest.map { request =>
  val negotiator = EncodingNegotiator(request.headers)
@@ -136,6 +143,12 @@ trait DeltaDirectives extends UriDirectives {
  ): Directive0 =
  conditionalCache(value, lastModified, mediaType, None, encoding)
 
+ /**
+ * Wraps its inner route with support for Conditional Requests as defined by http://tools.ietf.org/html/rfc7232
+ *
+ * Supports `Etag` and `Last-Modified` headers:
+ * https://doc.akka.io/docs/akka-http/10.0/routing-dsl/directives/cache-condition-directives/conditional.html
+ */
  def conditionalCache(
  value: Option[String],
  lastModified: Option[Instant],

delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/directives/EtagUtils.scala

@@ -14,6 +14,12 @@ object EtagUtils {
  encoding: HttpEncoding
  ) = s"${value}_${mediaType}${jsonldFormat.map { f => s"_$f" }.getOrElse("")}_$encoding"
 
+ /**
+ * Computes a `Etag` value by concatenating and hashing the provided values
+ *
+ * Note that the media type, the jsonld format and the encoding are present because they have an impact on the
+ * resource representation
+ */
  def compute(
  value: String,
  mediaType: MediaType,

delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/directives/FileResponse.scala

@@ -1,13 +1,16 @@
 package ch.epfl.bluebrain.nexus.delta.sdk.directives
 
-import akka.http.scaladsl.model.ContentType
+import akka.http.scaladsl.model.headers.`Content-Length`
+import akka.http.scaladsl.model.{ContentType, HttpHeader, StatusCode, StatusCodes}
 import cats.effect.IO
 import cats.syntax.all._
 import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.encoder.JsonLdEncoder
 import ch.epfl.bluebrain.nexus.delta.sdk.directives.FileResponse.{Content, Metadata}
-import ch.epfl.bluebrain.nexus.delta.sdk.{AkkaSource, JsonLdValue}
 import ch.epfl.bluebrain.nexus.delta.sdk.directives.Response.Complete
 import ch.epfl.bluebrain.nexus.delta.sdk.marshalling.HttpResponseFields
+import ch.epfl.bluebrain.nexus.delta.sdk.{AkkaSource, JsonLdValue}
+
+import java.time.Instant
 
 /**
  * A file response content
@@ -33,23 +36,55 @@ object FileResponse {
  * @param bytes
  * the file size
  */
- final case class Metadata(filename: String, contentType: ContentType, bytes: Option[Long])
+ final case class Metadata(
+ filename: String,
+ contentType: ContentType,
+ etag: Option[String],
+ lastModified: Option[Instant],
+ bytes: Option[Long]
+ )
+
+ object Metadata {
+ implicit def fileResponseMetadataHttpResponseFields: HttpResponseFields[Metadata] =
+ new HttpResponseFields[Metadata] {
+ override def statusFrom(value: Metadata): StatusCode = StatusCodes.OK
+ override def headersFrom(value: Metadata): Seq[HttpHeader] =
+ value.bytes.map { bytes => `Content-Length`(bytes) }.toSeq
+
+ override def entityTag(value: Metadata): Option[String] = value.etag
+
+ override def lastModified(value: Metadata): Option[Instant] = value.lastModified
+ }
+ }
 
  def apply[E: JsonLdEncoder: HttpResponseFields](
  filename: String,
  contentType: ContentType,
+ etag: Option[String],
+ lastModified: Option[Instant],
  bytes: Option[Long],
  io: IO[Either[E, AkkaSource]]
  ) =
  new FileResponse(
- Metadata(filename, contentType, bytes),
+ Metadata(filename, contentType, etag, lastModified, bytes),
  io.map { r =>
  r.leftMap { e =>
  Complete(e).map(JsonLdValue(_))
  }
  }
  )
 
- def apply(filename: String, contentType: ContentType, bytes: Option[Long], source: AkkaSource): FileResponse =
- new FileResponse(Metadata(filename, contentType, bytes), IO.pure(Right(source)))
+ def apply(
+ filename: String,
+ contentType: ContentType,
+ etag: Option[String],
+ lastModified: Option[Instant],
+ bytes: Option[Long],
+ source: AkkaSource
+ ): FileResponse =
+ new FileResponse(Metadata(filename, contentType, etag, lastModified, bytes), IO.pure(Right(source)))
+
+ def noCache(filename: String, contentType: ContentType, bytes: Option[Long], source: AkkaSource): FileResponse =
+ new FileResponse(Metadata(filename, contentType, None, None, bytes), IO.pure(Right(source)))
+
 }

delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/directives/ResponseToJsonLd.scala

@@ -16,6 +16,7 @@ import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.RemoteContextResolution
 import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.encoder.JsonLdEncoder
 import ch.epfl.bluebrain.nexus.delta.rdf.utils.JsonKeyOrdering
 import ch.epfl.bluebrain.nexus.delta.sdk.JsonLdValue
+import ch.epfl.bluebrain.nexus.delta.sdk.syntax._
 import ch.epfl.bluebrain.nexus.delta.sdk.directives.ResponseToJsonLd.{RouteOutcome, UseLeft, UseRight}
 import ch.epfl.bluebrain.nexus.delta.sdk.directives.DeltaDirectives._
 import ch.epfl.bluebrain.nexus.delta.sdk.directives.Response.{Complete, Reject}
@@ -149,10 +150,22 @@ object ResponseToJsonLd extends FileBytesInstances {
  case Right(Right((metadata, content))) =>
  headerValueByType(Accept) { accept =>
  if (accept.mediaRanges.exists(_.matches(metadata.contentType.mediaType))) {
- val encodedFilename = attachmentString(metadata.filename)
- respondWithHeaders(RawHeader("Content-Disposition", s"""attachment; filename="$encodedFilename"""")) {
- complete(statusOverride.getOrElse(OK), HttpEntity(metadata.contentType, content))
+ val encodedFilename = attachmentString(metadata.filename)
+ val contentDisposition =
+ RawHeader("Content-Disposition", s"""attachment; filename="$encodedFilename"""")
+ requestEncoding { encoding =>
+ conditionalCache(
+ metadata.entityTag,
+ metadata.lastModified,
+ metadata.contentType.mediaType,
+ encoding
+ ) {
+ respondWithHeaders(contentDisposition, metadata.headers: _*) {
+ complete(statusOverride.getOrElse(OK), HttpEntity(metadata.contentType, content))
+ }
+ }
  }
+
  } else
  reject(unacceptedMediaTypeRejection(Seq(metadata.contentType.mediaType)))
  }

delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/model/ResourceF.scala

@@ -255,10 +255,9 @@ object ResourceF {
  }
  }
 
+ def etagValue[A](value: ResourceF[A]) = s"${value.uris.relativeAccessUri}_${value.rev}"
+
  implicit def resourceFHttpResponseFields[A]: HttpResponseFields[ResourceF[A]] =
- HttpResponseFields.fromTagAndLastModified { value =>
- val etagValue = s"${value.uris.relativeAccessUri}_${value.rev}"
- (etagValue, value.updatedAt)
- }
+ HttpResponseFields.fromTagAndLastModified { value => (etagValue(value), value.updatedAt) }
 
 }

delta/sdk/src/test/scala/ch/epfl/bluebrain/nexus/delta/sdk/directives/ResponseToJsonLdSpec.scala

@@ -2,7 +2,7 @@ package ch.epfl.bluebrain.nexus.delta.sdk.directives
 
 import akka.http.scaladsl.model.ContentTypes.`text/plain(UTF-8)`
 import akka.http.scaladsl.model.MediaRanges.`*/*`
-import akka.http.scaladsl.model.headers.Accept
+import akka.http.scaladsl.model.headers.{`Content-Length`, Accept}
 import akka.http.scaladsl.model.{ContentType, StatusCodes}
 import akka.http.scaladsl.server.RouteConcatenation
 import akka.stream.scaladsl.Source
@@ -23,6 +23,8 @@ import ch.epfl.bluebrain.nexus.delta.sdk.{AkkaSource, SimpleRejection, SimpleRes
 import ch.epfl.bluebrain.nexus.delta.sourcing.model.ProjectRef
 import ch.epfl.bluebrain.nexus.testkit.scalatest.ce.CatsEffectSpec
 
+import java.time.Instant
+
 class ResponseToJsonLdSpec extends CatsEffectSpec with RouteHelpers with JsonSyntax with RouteConcatenation {
 
  implicit val rcr: RemoteContextResolution =
@@ -36,7 +38,8 @@ class ResponseToJsonLdSpec extends CatsEffectSpec with RouteHelpers with JsonSyn
  private def responseWithSourceError[E: JsonLdEncoder: HttpResponseFields](error: E) = {
  responseWith(
  `text/plain(UTF-8)`,
- IO.pure(Left(error))
+ IO.pure(Left(error)),
+ cacheable = false
  )
  }
 
@@ -48,13 +51,16 @@ class ResponseToJsonLdSpec extends CatsEffectSpec with RouteHelpers with JsonSyn
 
  private def responseWith[E: JsonLdEncoder: HttpResponseFields](
  contentType: ContentType,
- contents: IO[Either[E, AkkaSource]]
+ contents: IO[Either[E, AkkaSource]],
+ cacheable: Boolean
  ) = {
  IO.pure(
  Right(
  FileResponse(
  "file.name",
  contentType,
+ Option.when(cacheable)("test"),
+ Option.when(cacheable)(Instant.EPOCH),
  Some(1024L),
  contents
  )
@@ -70,11 +76,21 @@ class ResponseToJsonLdSpec extends CatsEffectSpec with RouteHelpers with JsonSyn
 
  "Return the contents of a file" in {
  request ~> emit(
- responseWith(`text/plain(UTF-8)`, fileSourceOfString(FileContents))
+ responseWith(`text/plain(UTF-8)`, fileSourceOfString(FileContents), cacheable = true)
  ) ~> check {
  status shouldEqual StatusCodes.OK
  contentType shouldEqual `text/plain(UTF-8)`
  response.asString shouldEqual FileContents
+ response.header[`Content-Length`].value shouldEqual `Content-Length`(1024L)
+ response.expectConditionalCacheHeaders
+ }
+ }
+
+ "Not return the conditional cache headers" in {
+ request ~> emit(
+ responseWith(`text/plain(UTF-8)`, fileSourceOfString(FileContents), cacheable = false)
+ ) ~> check {
+ response.expectNoConditionalCacheHeaders
  }
  }
 

delta/sdk/src/test/scala/ch/epfl/bluebrain/nexus/delta/sdk/utils/RouteHelpers.scala

@@ -102,6 +102,11 @@ final class HttpResponseOps(private val http: HttpResponse) extends Consumer {
  http.header[LastModified] shouldBe defined
  }
 
+ def expectNoConditionalCacheHeaders(implicit position: Position): Assertion = {
+ http.header[ETag] shouldBe empty
+ http.header[LastModified] shouldBe empty
+ }
+
 }
 
 final class HttpChunksOps(private val chunks: Source[ChunkStreamPart, Any]) extends Consumer {

tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/BaseIntegrationSpec.scala

@@ -1,6 +1,6 @@
 package ch.epfl.bluebrain.nexus.tests
 
-import akka.http.javadsl.model.headers.{HttpCredentials, LastModified}
+import akka.http.javadsl.model.headers.HttpCredentials
 import akka.http.scaladsl.model._
 import akka.http.scaladsl.model.headers._
 import akka.http.scaladsl.testkit.ScalatestRouteTest
@@ -214,11 +214,6 @@ trait BaseIntegrationSpec
  private[tests] def contentType(response: HttpResponse): ContentType =
  response.header[`Content-Type`].value.contentType
 
- private[tests] def expectConditionalCacheHeaders(response: HttpResponse)(implicit position: Position): Assertion = {
- response.header[ETag] shouldBe defined
- response.header[LastModified] shouldBe defined
- }
-
  private[tests] def genId(length: Int = 15): String =
  genString(length = length, Vector.range('a', 'z') ++ Vector.range('0', '9'))