From fb27b8052846fabedfb57a46129635c96f728e06 Mon Sep 17 00:00:00 2001 From: Bram Buitendijk Date: Tue, 22 Oct 2024 00:03:29 +0200 Subject: [PATCH] implement addMultiFieldContainerIndex --- .../resources/ContainerServiceResource.kt | 87 +++++++++++-------- .../huc/annorepo/resources/W3CResource.kt | 15 ++-- .../annorepo/resources/tools/IndexChore.kt | 4 +- .../annorepo/resources/tools/IndexManager.kt | 65 ++++++-------- .../huc/annorepo/service/MongoDbUpdater.kt | 11 ++- .../resources/ContainerServiceResourceTest.kt | 33 ++++++- 6 files changed, 123 insertions(+), 92 deletions(-) diff --git a/server/src/main/kotlin/nl/knaw/huc/annorepo/resources/ContainerServiceResource.kt b/server/src/main/kotlin/nl/knaw/huc/annorepo/resources/ContainerServiceResource.kt index 3c4779ff..96915ba4 100644 --- a/server/src/main/kotlin/nl/knaw/huc/annorepo/resources/ContainerServiceResource.kt +++ b/server/src/main/kotlin/nl/knaw/huc/annorepo/resources/ContainerServiceResource.kt @@ -443,9 +443,17 @@ class ContainerServiceResource( IndexType.entries.joinToString(", ") { it.name.lowercase() } }" ) + val indexParts = + listOf( + IndexManager.IndexPart( + fieldName = fieldNameParam, + indexType = indexType, + indexTypeName = indexTypeParam + ) + ) val indexChore = - indexManager.startIndexCreation(containerName, fieldNameParam, indexTypeParam, indexType) - indexManager.getIndexChore(containerName, fieldNameParam, indexTypeParam) + indexManager.startIndexCreation(containerName, indexParts) +// indexManager.getIndexChore(containerName, indexParts) val location = uriFactory.singleFieldIndexURL(containerName, fieldNameParam, indexTypeParam) return Response.created(location) .link(uriFactory.indexStatusURL(containerName, fieldNameParam, indexTypeParam), "status") @@ -453,45 +461,43 @@ class ContainerServiceResource( .build() } -// @OptIn(ExperimentalStdlibApi::class) -// @Operation(description = "Add a multi-field index") -// @Timed -// @POST -// @Path("{containerName}/$INDEXES") -// fun addMultiFieldContainerIndex( -// @PathParam("containerName") containerName: String, -// multiFieldIndexSettings: Map, -// @Context context: SecurityContext, -// ): Response { -// context.checkUserHasAdminRightsInThisContainer(containerName) -// val indexParts = multiFieldIndexSettings.map { (fieldName, indexTypeParam) -> -// val indexType = -// IndexType.fromString(indexTypeParam) ?: throw BadRequestException( -// "Unknown indexType $indexTypeParam; expected indexTypes: ${ -// IndexType.entries.joinToString(", ") { it.name.lowercase() } -// }" -// ) -// IndexPart(fieldName, indexTypeParam, indexType) -// } -// val indexChore = -// indexManager.startIndexCreation(containerName, indexParts) -//// indexManager.getIndexChore(containerName, fieldNameParam, indexTypeParam) -// val indexName = indexName(multiFieldIndexSettings) -// val location = uriFactory.multiFieldIndexURL(containerName, indexName) -// return Response.created(location) + @OptIn(ExperimentalStdlibApi::class) + @Operation(description = "Add a multi-field index") + @Timed + @POST + @Path("{containerName}/$INDEXES") + fun addMultiFieldContainerIndex( + @PathParam("containerName") containerName: String, + multiFieldIndexSettings: Map, + @Context context: SecurityContext, + ): Response { + context.checkUserHasAdminRightsInThisContainer(containerName) + val indexParts = multiFieldIndexSettings.map { (fieldName, indexTypeParam) -> + val indexType = + IndexType.fromString(indexTypeParam) ?: throw BadRequestException( + "Unknown indexType $indexTypeParam; expected indexTypes: ${ + IndexType.entries.joinToString(", ") { it.name.lowercase() } + }" + ) + IndexManager.IndexPart(fieldName, indexTypeParam, indexType) + } + val indexChore = + indexManager.startIndexCreation(containerName, indexParts) +// indexManager.getIndexChore(containerName, fieldNameParam, indexTypeParam) + val indexName = indexName(multiFieldIndexSettings) + val location = uriFactory.multiFieldIndexURL(containerName, indexName) + return Response.created(location) // .link(uriFactory.indexStatusURL(containerName, fieldNameParam, indexTypeParam), "status") -// .entity(indexChore.status.summary()) -// .build() -// } + .entity(indexChore.status.summary()) + .build() + } private fun indexName(multiFieldIndexSettings: Map): String { - return multiFieldIndexSettings.entries.map { (fieldName, indexType) -> + return multiFieldIndexSettings.entries.joinToString("-") { (fieldName, indexType) -> "${fieldName}_${indexType.lowercase()}" - }.joinToString("-") + } } - data class IndexPart(val fieldName: String, val indexTypeParam: String, val indexType: IndexType) - @Operation(description = "Get an index definition") @Timed @GET @@ -522,7 +528,16 @@ class ContainerServiceResource( ): Response { context.checkUserHasAdminRightsInThisContainer(containerName) - val indexChore = indexManager.getIndexChore(containerName, fieldName, indexType) ?: throw NotFoundException() + val indexChore = indexManager.getIndexChore( + containerName, + listOf( + IndexManager.IndexPart( + fieldName = fieldName, + indexTypeName = indexType, + indexType = IndexType.valueOf(indexType) + ) + ) + ) ?: throw NotFoundException() return Response.ok(indexChore.status.summary()).build() } diff --git a/server/src/main/kotlin/nl/knaw/huc/annorepo/resources/W3CResource.kt b/server/src/main/kotlin/nl/knaw/huc/annorepo/resources/W3CResource.kt index dc6dc2ed..fecb5f72 100644 --- a/server/src/main/kotlin/nl/knaw/huc/annorepo/resources/W3CResource.kt +++ b/server/src/main/kotlin/nl/knaw/huc/annorepo/resources/W3CResource.kt @@ -81,6 +81,8 @@ class W3CResource( private val indexManager: IndexManager ) : AbstractContainerResource(configuration, containerDAO, ContainerAccessChecker(containerUserDAO)) { + private val paginationStage = Aggregates.limit(configuration.pageSize) + @Operation(description = "Create an Annotation Container") @Timed @POST @@ -105,10 +107,13 @@ class W3CResource( } indexManager.startIndexCreation( containerName = containerName, - fieldName = ANNOTATION_NAME_FIELD, - isJsonField = false, - indexTypeName = "annotation_name", - indexType = IndexType.HASHED + indexParts = listOf( + IndexManager.IndexPart( + fieldName = ANNOTATION_NAME_FIELD, + indexTypeName = "annotation_name", + indexType = IndexType.HASHED + ) + ) ) val containerData = getContainerPage(containerName, 0, configuration.pageSize) @@ -388,7 +393,6 @@ class W3CResource( return jo.toMap() } - private val paginationStage = Aggregates.limit(configuration.pageSize) private fun validateETag(req: Request, eTag: EntityTag) { try { req.evaluatePreconditions(eTag) ?: throw PreconditionFailedException() @@ -434,7 +438,6 @@ class W3CResource( } private fun lastPage(count: Long, pageSize: Int) = (count - 1).div(pageSize).toInt() - private fun toAnnotationMap(a: Document, containerName: String): WebAnnotationAsMap { return a.get(ANNOTATION_FIELD, Document::class.java) .toMutableMap() diff --git a/server/src/main/kotlin/nl/knaw/huc/annorepo/resources/tools/IndexChore.kt b/server/src/main/kotlin/nl/knaw/huc/annorepo/resources/tools/IndexChore.kt index 8f1d36b7..8e4a7e19 100644 --- a/server/src/main/kotlin/nl/knaw/huc/annorepo/resources/tools/IndexChore.kt +++ b/server/src/main/kotlin/nl/knaw/huc/annorepo/resources/tools/IndexChore.kt @@ -14,7 +14,7 @@ import nl.knaw.huc.annorepo.api.ChoreStatusSummary class IndexChore( val id: String, private val container: MongoCollection, - private val fieldName: String, + private val fieldNames: List, private val index: Bson ) : Runnable { @@ -49,7 +49,7 @@ class IndexChore( status.state = State.RUNNING status.startTime = Instant.now() try { - val partialFilter = Filters.exists(fieldName) + val partialFilter = Filters.or(fieldNames.map{Filters.exists(it)}) val indexName = container.createIndex(index, IndexOptions().partialFilterExpression(partialFilter)) // val indexName = container.createIndex(index, IndexOptions().partialFilterExpression(partialFilter)) logger.info { "created index: $indexName" } diff --git a/server/src/main/kotlin/nl/knaw/huc/annorepo/resources/tools/IndexManager.kt b/server/src/main/kotlin/nl/knaw/huc/annorepo/resources/tools/IndexManager.kt index 72d9393a..d8e1ddb0 100644 --- a/server/src/main/kotlin/nl/knaw/huc/annorepo/resources/tools/IndexManager.kt +++ b/server/src/main/kotlin/nl/knaw/huc/annorepo/resources/tools/IndexManager.kt @@ -11,63 +11,48 @@ import nl.knaw.huc.annorepo.api.IndexType.TEXT import nl.knaw.huc.annorepo.dao.ContainerDAO class IndexManager(val containerDAO: ContainerDAO) { + data class IndexPart( + val fieldName: String, + val indexTypeName: String, + val indexType: IndexType, + val isJsonField: Boolean = false + ) - // fun startIndexCreation( -// containerName: String, -// indexParts: List -// ): IndexChore { -// val container = containerDAO.getCollection(containerName) -// val fullFieldName = if (isJsonField) "${ARConst.ANNOTATION_FIELD}.${fieldName}" else fieldName -// Indexes. -// val index = when (indexType) { -// HASHED -> Indexes.hashed(fullFieldName) -// ASCENDING -> Indexes.ascending(fullFieldName) -// DESCENDING -> Indexes.descending(fullFieldName) -// TEXT -> Indexes.text(fieldName) -// else -> throw RuntimeException("Cannot make an index with type $indexType") -// } -// return startIndexChore( -// IndexChore( -// id = choreId(containerName, fieldName, indexTypeName), -// container = container, -// fieldName = fullFieldName, -// index = index -// ) -// ) -// } fun startIndexCreation( containerName: String, - fieldName: String, - indexTypeName: String, - indexType: IndexType, - isJsonField: Boolean = true + indexParts: List ): IndexChore { val container = containerDAO.getCollection(containerName) - val fullFieldName = if (isJsonField) "${ARConst.ANNOTATION_FIELD}.${fieldName}" else fieldName - val index = when (indexType) { - HASHED -> Indexes.hashed(fullFieldName) - ASCENDING -> Indexes.ascending(fullFieldName) - DESCENDING -> Indexes.descending(fullFieldName) - TEXT -> Indexes.text(fieldName) - else -> throw RuntimeException("Cannot make an index with type $indexType") + val fullFieldNames = mutableListOf() + val indexes = indexParts.map { + val fullFieldName = if (it.isJsonField) "${ARConst.ANNOTATION_FIELD}.${it.fieldName}" else it.fieldName + fullFieldNames.add(fullFieldName) + when (it.indexType) { + HASHED -> Indexes.hashed(fullFieldName) + ASCENDING -> Indexes.ascending(fullFieldName) + DESCENDING -> Indexes.descending(fullFieldName) + TEXT -> Indexes.text(fullFieldName) + else -> throw RuntimeException("Cannot make an index with type $it.indexType on field $fullFieldName") + } } + val index = Indexes.compoundIndex(indexes) return startIndexChore( IndexChore( - id = choreId(containerName, fieldName, indexTypeName), + id = choreId(containerName, indexParts), container = container, - fieldName = fullFieldName, + fieldNames = fullFieldNames, index = index ) ) } - fun getIndexChore(containerName: String, fieldName: String, indexTypeName: String): IndexChore? { - val id = choreId(containerName, fieldName, indexTypeName) + fun getIndexChore(containerName: String, indexParts: List): IndexChore? { + val id = choreId(containerName, indexParts) return IndexChoreIndex[id] } - private fun choreId(containerName: String, fieldName: String, indexTypeName: String) = - "$containerName/$fieldName/$indexTypeName".lowercase() + private fun choreId(containerName: String, indexParts: List) = + ("$containerName/" + indexParts.joinToString("/") { "${it.fieldName}/${it.indexType}" }).lowercase() private fun startIndexChore(chore: IndexChore): IndexChore { IndexChoreIndex[chore.id] = chore diff --git a/server/src/main/kotlin/nl/knaw/huc/annorepo/service/MongoDbUpdater.kt b/server/src/main/kotlin/nl/knaw/huc/annorepo/service/MongoDbUpdater.kt index 2a107b8c..30fbc488 100644 --- a/server/src/main/kotlin/nl/knaw/huc/annorepo/service/MongoDbUpdater.kt +++ b/server/src/main/kotlin/nl/knaw/huc/annorepo/service/MongoDbUpdater.kt @@ -57,10 +57,13 @@ class MongoDbUpdater( logger.info { "> creating annotation_name index" } indexManager.startIndexCreation( containerName = containerName, - fieldName = ANNOTATION_NAME_FIELD, - isJsonField = false, - indexTypeName = "annotation_name", - indexType = IndexType.HASHED + indexParts = listOf( + IndexManager.IndexPart( + fieldName = ANNOTATION_NAME_FIELD, + indexType = IndexType.HASHED, + indexTypeName = "annotation_name)" + ) + ) ) } } diff --git a/server/src/test/kotlin/nl/knaw/huc/annorepo/resources/ContainerServiceResourceTest.kt b/server/src/test/kotlin/nl/knaw/huc/annorepo/resources/ContainerServiceResourceTest.kt index a59d9a25..981b292c 100644 --- a/server/src/test/kotlin/nl/knaw/huc/annorepo/resources/ContainerServiceResourceTest.kt +++ b/server/src/test/kotlin/nl/knaw/huc/annorepo/resources/ContainerServiceResourceTest.kt @@ -117,7 +117,7 @@ class ContainerServiceResourceTest { """.trimIndent() useEditorUser() val response = resource.createSearch(CONTAINER_NAME, queryJson, context = securityContext) - logger.info { "result=$response"} + logger.info { "result=$response" } val locations = response.headers["location"] as List<*> val location: URI = locations[0] as URI @@ -241,7 +241,8 @@ class ContainerServiceResourceTest { assertRoleAuthorizationForBlock( authorizedRoles = setOf(Role.ROOT, Role.ADMIN) ) { - val response = resource.addSingleFieldContainerIndex(CONTAINER_NAME, "fieldName", "indexType", securityContext) + val response = + resource.addSingleFieldContainerIndex(CONTAINER_NAME, "fieldName", "indexType", securityContext) assertNotNull(response) } } @@ -255,7 +256,12 @@ class ContainerServiceResourceTest { authorizedRoles = setOf(Role.ROOT, Role.ADMIN) ) { val response = - resource.getSingleFieldContainerIndexDefinition(CONTAINER_NAME, "fieldName", "indexType", securityContext) + resource.getSingleFieldContainerIndexDefinition( + CONTAINER_NAME, + "fieldName", + "indexType", + securityContext + ) assertNotNull(response) } } @@ -269,7 +275,26 @@ class ContainerServiceResourceTest { authorizedRoles = setOf(Role.ROOT, Role.ADMIN) ) { val response = - resource.deleteSingleFieldContainerIndex(CONTAINER_NAME, "fieldName", "indexType", securityContext) + resource.deleteSingleFieldContainerIndex( + CONTAINER_NAME, + "fieldName", + "indexType", + securityContext + ) + assertNotNull(response) + } + } + } + + @Nested + inner class AddMultiFieldContainerIndexTest { + @Test + fun `addMultiFieldContainerIndex endpoint can be used by root or admin, but not by others`() { + assertRoleAuthorizationForBlock( + authorizedRoles = setOf(Role.ROOT, Role.ADMIN) + ) { + val response = + resource.addMultiFieldContainerIndex(CONTAINER_NAME, mapOf(), securityContext) assertNotNull(response) } }