Allow to add exceptions to schema enforcement (#5075)

Co-authored-by: Simon Dumas <[email protected]>

delta/app/src/main/resources/app.conf

@@ -269,6 +269,11 @@ app {
  event-log = ${app.defaults.event-log}
  # Reject payloads which contain nexus metadata fields (any field beginning with _)
  decoding-option = "strict"
+ # Defines exceptions for schema enforcement
+ schema-enforcement {
+ type-whitelist = []
+ allow-no-types = false
+ }
  # Do not create a new revision of a resource when the update does not introduce a change
  skip-update-no-change = true
  }

delta/app/src/main/scala/ch/epfl/bluebrain/nexus/delta/wiring/ResourcesModule.scala

@@ -37,8 +37,10 @@ object ResourcesModule extends ModuleDef {
  ResourceResolution.schemaResource(aclCheck, resolvers, fetchSchema, excludeDeprecated = false)
  }
 
- make[ValidateResource].from { (resourceResolution: ResourceResolution[Schema], validateShacl: ValidateShacl) =>
- ValidateResource(resourceResolution, validateShacl)
+ make[ValidateResource].from {
+ (resourceResolution: ResourceResolution[Schema], validateShacl: ValidateShacl, config: ResourcesConfig) =>
+ val schemaClaimResolver = SchemaClaimResolver(resourceResolution, config.schemaEnforcement)
+ ValidateResource(schemaClaimResolver, validateShacl)
  }
 
  make[ResourcesConfig].from { (config: AppConfig) => config.resources }

delta/app/src/test/resources/schemas/errors/invalid-schema-2.json

@@ -7,19 +7,6 @@
  "reason": "Schema 'https://bluebrain.github.io/nexus/vocabulary/pretendschema' could not be resolved in 'myorg/myproject'",
  "report": {
  "history": [
- {
- "rejections": [
- {
- "cause": {
- "@type": "ResourceNotFound",
- "reason": "The resource was not found in project 'myorg/myproject'."
- },
- "project": "myorg/myproject"
- }
- ],
- "resolverId": "https://bluebrain.github.io/nexus/vocabulary/in-project",
- "success": false
- }
  ]
  }
 }

delta/app/src/test/scala/ch/epfl/bluebrain/nexus/delta/routes/ResourcesRoutesSpec.scala

@@ -4,41 +4,37 @@ import akka.http.scaladsl.model.MediaTypes.`text/html`
 import akka.http.scaladsl.model.headers.{Accept, Location, OAuth2BearerToken, RawHeader}
 import akka.http.scaladsl.model.{RequestEntity, StatusCodes, Uri}
 import akka.http.scaladsl.server.Route
-import cats.effect.IO
 import cats.implicits._
 import ch.epfl.bluebrain.nexus.delta.kernel.utils.{UUIDF, UrlUtils}
 import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri
 import ch.epfl.bluebrain.nexus.delta.rdf.Vocabulary.{contexts, nxv, schema => schemaOrg, schemas}
 import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.JsonLdContext.keywords
-import ch.epfl.bluebrain.nexus.delta.rdf.shacl.ValidateShacl
 import ch.epfl.bluebrain.nexus.delta.sdk.IndexingAction
 import ch.epfl.bluebrain.nexus.delta.sdk.acls.AclSimpleCheck
 import ch.epfl.bluebrain.nexus.delta.sdk.acls.model.AclAddress
-import ch.epfl.bluebrain.nexus.delta.sdk.generators.{ProjectGen, ResourceResolutionGen, SchemaGen}
+import ch.epfl.bluebrain.nexus.delta.sdk.generators.ProjectGen
 import ch.epfl.bluebrain.nexus.delta.sdk.identities.IdentitiesDummy
 import ch.epfl.bluebrain.nexus.delta.sdk.identities.model.Caller
 import ch.epfl.bluebrain.nexus.delta.sdk.implicits._
-import ch.epfl.bluebrain.nexus.delta.sdk.model.Fetch.FetchF
 import ch.epfl.bluebrain.nexus.delta.sdk.model.{IdSegmentRef, ResourceUris}
 import ch.epfl.bluebrain.nexus.delta.sdk.permissions.Permissions.resources
 import ch.epfl.bluebrain.nexus.delta.sdk.projects.FetchContextDummy
 import ch.epfl.bluebrain.nexus.delta.sdk.projects.model.ApiMappings
 import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.ResolverContextResolution
 import ch.epfl.bluebrain.nexus.delta.sdk.resources.NexusSource.DecodingOption
 import ch.epfl.bluebrain.nexus.delta.sdk.resources._
-import ch.epfl.bluebrain.nexus.delta.sdk.schemas.model.Schema
 import ch.epfl.bluebrain.nexus.delta.sdk.utils.BaseRouteSpec
 import ch.epfl.bluebrain.nexus.delta.sourcing.ScopedEventLog
 import ch.epfl.bluebrain.nexus.delta.sourcing.model.Identity.{Anonymous, Authenticated, Group, Subject, User}
+import ch.epfl.bluebrain.nexus.delta.sourcing.model.ProjectRef
 import ch.epfl.bluebrain.nexus.delta.sourcing.model.Tag.UserTag
-import ch.epfl.bluebrain.nexus.delta.sourcing.model.{ProjectRef, ResourceRef}
 import ch.epfl.bluebrain.nexus.testkit.scalatest.ce.CatsIOValues
 import io.circe.{Json, Printer}
 import org.scalatest.Assertion
 
 import java.util.UUID
 
-class ResourcesRoutesSpec extends BaseRouteSpec with CatsIOValues {
+class ResourcesRoutesSpec extends BaseRouteSpec with ValidateResourceFixture with CatsIOValues {
 
  private val uuid = UUID.randomUUID()
  implicit private val uuidF: UUIDF = UUIDF.fixed(uuid)
@@ -54,9 +50,9 @@ class ResourcesRoutesSpec extends BaseRouteSpec with CatsIOValues {
  private val asReader = addCredentials(OAuth2BearerToken("reader"))
  private val asWriter = addCredentials(OAuth2BearerToken("writer"))
 
- private val am  = ApiMappings("nxv" -> nxv.base, "Person" -> schemaOrg.Person)
- private val projBase  = nxv.base
- private val project  = ProjectGen.resourceFor(
+ private val am = ApiMappings("nxv" -> nxv.base, "Person" -> schemaOrg.Person)
+ private val projBase = nxv.base
+ private val project = ProjectGen.resourceFor(
  ProjectGen.project(
  "myorg",
  "myproject",
@@ -66,12 +62,10 @@ class ResourcesRoutesSpec extends BaseRouteSpec with CatsIOValues {
  mappings = am + Resources.mappings
  )
  )
- private val projectRef = project.value.ref
- private val schemaSource = jsonContentOf("resources/schema.json").addContext(contexts.shacl, contexts.schemasMetadata)
- private val schema1 = SchemaGen.schema(nxv + "myschema", project.value.ref, schemaSource.removeKeys(keywords.id))
- private val schema2 = SchemaGen.schema(schemaOrg.Person, project.value.ref, schemaSource.removeKeys(keywords.id))
- private val schema3 = SchemaGen.schema(nxv + "otherSchema", project.value.ref, schemaSource.removeKeys(keywords.id))
- private val tag = UserTag.unsafe("mytag")
+ private val projectRef = project.value.ref
+ private val schema1 = nxv + "myschema"
+ private val schema2 = nxv + "otherSchema"
+ private val tag = UserTag.unsafe("mytag")
 
  private val myId = nxv + "myid" // Resource created against no schema with id present on the payload
  private def encodeWithBase(id: String) = UrlUtils.encode((nxv + id).toString)
@@ -89,27 +83,13 @@ class ResourcesRoutesSpec extends BaseRouteSpec with CatsIOValues {
 
  private val aclCheck = AclSimpleCheck().accepted
 
- private val fetchSchema: (ResourceRef, ProjectRef) => FetchF[Schema] = {
- case (ref, _) if ref.iri == schema2.id => IO.pure(Some(SchemaGen.resourceFor(schema2, deprecated = true)))
- case (ref, _) if ref.iri == schema1.id => IO.pure(Some(SchemaGen.resourceFor(schema1)))
- case (ref, _) if ref.iri == schema3.id => IO.pure(Some(SchemaGen.resourceFor(schema3)))
- case _ => IO.none
- }
-
- private val validator: ValidateResource = ValidateResource(
- ResourceResolutionGen.singleInProject(projectRef, fetchSchema),
- ValidateShacl(rcr).accepted
- )
+ private val validateResource = validateFor(Set((projectRef, schema1), (projectRef, schema2)))
  private val fetchContext = FetchContextDummy(List(project.value))
  private val resolverContextResolution: ResolverContextResolution = ResolverContextResolution(rcr)
 
  private def routesWithDecodingOption(implicit decodingOption: DecodingOption): (Route, Resources) = {
- val resourceDef = Resources.definition(validator, DetectChange(enabled = true), clock)
- val scopedLog = ScopedEventLog(
- resourceDef,
- ResourcesConfig(eventLogConfig, decodingOption, skipUpdateNoChange = true).eventLog,
- xas
- )
+ val resourceDef = Resources.definition(validateResource, DetectChange(enabled = true), clock)
+ val scopedLog = ScopedEventLog(resourceDef, eventLogConfig, xas)
 
  val resources = ResourcesImpl(
  scopedLog,
@@ -158,7 +138,7 @@ class ResourcesRoutesSpec extends BaseRouteSpec with CatsIOValues {
  "create a resource" in {
  val endpoints = List(
  ("/v1/resources/myorg/myproject", schemas.resources),
- ("/v1/resources/myorg/myproject/myschema", schema1.id)
+ ("/v1/resources/myorg/myproject/myschema", schema1)
  )
  forAll(endpoints) { case (endpoint, schema) =>
  val id = genString()
@@ -172,7 +152,7 @@ class ResourcesRoutesSpec extends BaseRouteSpec with CatsIOValues {
  "create a tagged resource" in {
  val endpoints = List(
  ("/v1/resources/myorg/myproject?tag=mytag", schemas.resources),
- ("/v1/resources/myorg/myproject/myschema?tag=mytag", schema1.id)
+ ("/v1/resources/myorg/myproject/myschema?tag=mytag", schema1)
  )
  val (routes, resources) = routesWithDecodingOption(DecodingOption.Strict)
  forAll(endpoints) { case (endpoint, schema) =>
@@ -188,7 +168,7 @@ class ResourcesRoutesSpec extends BaseRouteSpec with CatsIOValues {
  "create a resource with an authenticated user and provided id" in {
  val endpoints = List(
  ((id: String) => s"/v1/resources/myorg/myproject/_/$id", schemas.resources),
- ((id: String) => s"/v1/resources/myorg/myproject/myschema/$id", schema1.id)
+ ((id: String) => s"/v1/resources/myorg/myproject/myschema/$id", schema1)
  )
  forAll(endpoints) { case (endpoint, schema) =>
  val id = genString()
@@ -202,7 +182,7 @@ class ResourcesRoutesSpec extends BaseRouteSpec with CatsIOValues {
  "create a tagged resource with an authenticated user and provided id" in {
  val endpoints = List(
  ((id: String) => s"/v1/resources/myorg/myproject/_/$id?tag=mytag", schemas.resources),
- ((id: String) => s"/v1/resources/myorg/myproject/myschema/$id?tag=mytag", schema1.id)
+ ((id: String) => s"/v1/resources/myorg/myproject/myschema/$id?tag=mytag", schema1)
  )
  val (routes, resources) = routesWithDecodingOption(DecodingOption.Strict)
  forAll(endpoints) { case (endpoint, schema) =>
@@ -228,17 +208,6 @@ class ResourcesRoutesSpec extends BaseRouteSpec with CatsIOValues {
  }
  }
 
- "fail to create a resource that does not validate against a schema" in {
- val payloadFailingSchemaConstraints = payloadWithoutId.replaceKeyWithValue("number", "wrong")
- Put(
- "/v1/resources/myorg/myproject/nxv:myschema/wrong",
- payloadFailingSchemaConstraints.toEntity
- ) ~> asWriter ~> routes ~> check {
- response.status shouldEqual StatusCodes.BadRequest
- response.asJson shouldEqual jsonContentOf("resources/errors/invalid-resource.json")
- }
- }
-
  "fail to create a resource against a schema that does not exist" in {
  Put(
  "/v1/resources/myorg/myproject/pretendschema/someid",
@@ -296,7 +265,7 @@ class ResourcesRoutesSpec extends BaseRouteSpec with CatsIOValues {
  forAll(endpoints) { case (endpoint, rev) =>
  Put(s"$endpoint?rev=$rev", payloadUpdated(id).toEntity(Printer.noSpaces)) ~> asWriter ~> routes ~> check {
  status shouldEqual StatusCodes.OK
- response.asJson shouldEqual standardWriterMetadata(id, rev = rev + 1, schema1.id)
+ response.asJson shouldEqual standardWriterMetadata(id, rev = rev + 1, schema1)
  }
  }
  }
@@ -341,7 +310,7 @@ class ResourcesRoutesSpec extends BaseRouteSpec with CatsIOValues {
  forAll(endpoints) { endpoint =>
  Put(s"$endpoint", payloadUpdated.toEntity(Printer.noSpaces)) ~> asWriter ~> routes ~> check {
  status shouldEqual StatusCodes.OK
- response.asJson shouldEqual standardWriterMetadata(id, rev = 1, schema = schema1.id)
+ response.asJson shouldEqual standardWriterMetadata(id, rev = 1, schema = schema1)
  }
  }
  }
@@ -384,7 +353,7 @@ class ResourcesRoutesSpec extends BaseRouteSpec with CatsIOValues {
  givenAResourceWithSchema("myschema") { id =>
  Put(s"/v1/resources/$projectRef/otherSchema/$id/update-schema") ~> asWriter ~> routes ~> check {
  response.status shouldEqual StatusCodes.OK
- response.asJson.hcursor.get[String]("_constrainedBy").toOption should contain(schema3.id.toString)
+ response.asJson.hcursor.get[String]("_constrainedBy").toOption should contain(schema2.toString)
  }
  }
  }
@@ -509,7 +478,7 @@ class ResourcesRoutesSpec extends BaseRouteSpec with CatsIOValues {
  s"/v1/resources/myorg/myproject/_/$id?rev=1",
  s"/v1/resources/myorg/myproject/$mySchema/$id?tag=$myTag"
  )
- val meta = standardWriterMetadata(id, schema = schema1.id, tpe = "schema:Custom")
+ val meta = standardWriterMetadata(id, schema = schema1, tpe = "schema:Custom")
  forAll(endpoints) { endpoint =>
  Get(endpoint) ~> asReader ~> routes ~> check {
  status shouldEqual StatusCodes.OK

delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/resources/ResourcesConfig.scala

@@ -1,6 +1,8 @@
 package ch.epfl.bluebrain.nexus.delta.sdk.resources
 
+import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri
 import ch.epfl.bluebrain.nexus.delta.sdk.resources.NexusSource.DecodingOption
+import ch.epfl.bluebrain.nexus.delta.sdk.resources.ResourcesConfig.SchemaEnforcementConfig
 import ch.epfl.bluebrain.nexus.delta.sourcing.config.EventLogConfig
 import pureconfig.ConfigReader
 import pureconfig.generic.semiauto.deriveReader
@@ -12,12 +14,31 @@ import pureconfig.generic.semiauto.deriveReader
  * configuration of the event log
  * @param decodingOption
  * strict/lenient decoding of resources
+ * @param schemaEnforcement
+ * configuration related to schema enforcement
  * @param skipUpdateNoChange
  * do not create a new revision when the update does not introduce a change in the current resource state
  */
-final case class ResourcesConfig(eventLog: EventLogConfig, decodingOption: DecodingOption, skipUpdateNoChange: Boolean)
+final case class ResourcesConfig(
+ eventLog: EventLogConfig,
+ decodingOption: DecodingOption,
+ schemaEnforcement: SchemaEnforcementConfig,
+ skipUpdateNoChange: Boolean
+)
 
 object ResourcesConfig {
- implicit final val resourcesConfigReader: ConfigReader[ResourcesConfig] =
+
+ /**
+ * Configuration to allow to bypass schema enforcing in some cases
+ * @param typeWhitelist
+ * types for which a schema is not required
+ * @param allowNoTypes
+ * allow to skip schema validation for resources without any types
+ */
+ final case class SchemaEnforcementConfig(typeWhitelist: Set[Iri], allowNoTypes: Boolean)
+
+ implicit final val resourcesConfigReader: ConfigReader[ResourcesConfig] = {
+ implicit val schemaEnforcementReader: ConfigReader[SchemaEnforcementConfig] = deriveReader[SchemaEnforcementConfig]
  deriveReader[ResourcesConfig]
+ }
 }

delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/resources/SchemaClaim.scala

@@ -3,9 +3,6 @@ package ch.epfl.bluebrain.nexus.delta.sdk.resources
 import cats.effect.IO
 import ch.epfl.bluebrain.nexus.delta.rdf.Vocabulary.schemas
 import ch.epfl.bluebrain.nexus.delta.sdk.identities.model.Caller
-import ch.epfl.bluebrain.nexus.delta.sdk.resources.SchemaClaim._
-import ch.epfl.bluebrain.nexus.delta.sdk.resources.ValidationResult._
-import ch.epfl.bluebrain.nexus.delta.sdk.resources.model.ResourceRejection.SchemaIsMandatory
 import ch.epfl.bluebrain.nexus.delta.sourcing.model.{ProjectRef, ResourceRef}
 
 /**
@@ -16,32 +13,8 @@ import ch.epfl.bluebrain.nexus.delta.sourcing.model.{ProjectRef, ResourceRef}
  */
 sealed trait SchemaClaim {
 
- /**
- * Validate the claim
- * @param enforceSchema
- * to ban unconstrained resources
- * @param submitOnDefinedSchema
- * the function to call when a schema is defined
- */
- def validate(enforceSchema: Boolean)(submitOnDefinedSchema: SubmitOnDefinedSchema): IO[ValidationResult] =
- this match {
- case CreateWithSchema(project, schema, caller) =>
- submitOnDefinedSchema(project, schema, caller)
- case CreateUnconstrained(project) =>
- onUnconstrained(project, enforceSchema)
- case UpdateToSchema(project, schema, caller) =>
- submitOnDefinedSchema(project, schema, caller)
- case UpdateToUnconstrained(project) =>
- onUnconstrained(project, enforceSchema)
- case KeepUnconstrained(project) =>
- IO.pure(NoValidation(project))
- }
-
  def project: ProjectRef
 
- private def onUnconstrained(project: ProjectRef, enforceSchema: Boolean) =
- IO.raiseWhen(enforceSchema)(SchemaIsMandatory(project)).as(NoValidation(project))
-
 }
 
 object SchemaClaim {
@@ -52,16 +25,16 @@ object SchemaClaim {
  def schemaRef: ResourceRef
  }
 
- final private case class CreateWithSchema(project: ProjectRef, schemaRef: ResourceRef, caller: Caller)
+ final case class CreateWithSchema(project: ProjectRef, schemaRef: ResourceRef, caller: Caller)
  extends DefinedSchemaClaim
- final private case class CreateUnconstrained(project: ProjectRef) extends SchemaClaim
+ final case class CreateUnconstrained(project: ProjectRef) extends SchemaClaim
 
- final private case class UpdateToSchema(project: ProjectRef, schemaRef: ResourceRef, caller: Caller)
+ final case class UpdateToSchema(project: ProjectRef, schemaRef: ResourceRef, caller: Caller)
  extends DefinedSchemaClaim
 
- final private case class UpdateToUnconstrained(project: ProjectRef) extends SchemaClaim
+ final case class UpdateToUnconstrained(project: ProjectRef) extends SchemaClaim
 
- final private case class KeepUnconstrained(project: ProjectRef) extends SchemaClaim
+ final case class KeepUnconstrained(project: ProjectRef) extends SchemaClaim
 
  private def isUnconstrained(schema: ResourceRef): Boolean = schema.iri == schemas.resources