From 2abc6b0fddf871f0746dba2d907978761e0c6eb4 Mon Sep 17 00:00:00 2001 From: Johannes Wienke Date: Thu, 27 Jun 2024 17:45:49 +0200 Subject: [PATCH] feat(model-server): implement BranchV1 for branches This adds a new media type for requesting branch metadata. The commit also prepares the openapi-generator templates to support media-type versioning by optionally being able to declare separate handler methods per media type. For registering types for JSON serialization, the model needs to be annotated with the intended media type via the x-modelix-media-type vendor extension. --- gradle/libs.versions.toml | 4 ++ .../specifications/model-server-v2.yaml | 43 ++++++++++-- model-server/build.gradle.kts | 2 + .../kotlin/org/modelix/model/server/Main.kt | 2 + .../server/handlers/ModelReplicationServer.kt | 19 ++++- .../openapi/templates/Paths.kt.mustache | 21 ++++++ .../resources/openapi/templates/api.mustache | 57 ++++++++++++++- .../model/server/ModelServerTestUtil.kt | 6 +- .../handlers/ModelReplicationServerTest.kt | 69 ++++++++++++++++++- 9 files changed, 211 insertions(+), 12 deletions(-) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 06bedfb84a..8c3c6c7f35 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -82,6 +82,10 @@ ktor-serialization = { group = "io.ktor", name = "ktor-serialization", version.r ktor-serialization-json = { group = "io.ktor", name = "ktor-serialization-kotlinx-json", version.ref = "ktor" } keycloak-authz-client = { group = "org.keycloak", name = "keycloak-authz-client", version = "25.0.1" } + +kotest-assertions-coreJvm = { group = "io.kotest", name = "kotest-assertions-core-jvm", version = "5.9.1" } +kotest-assertions-ktor = { group = "io.kotest.extensions", name = "kotest-assertions-ktor", version = "2.0.0" } + guava = { group = "com.google.guava", name = "guava", version = "33.2.1-jre" } org-json = { group = "org.json", name = "json", version = "20240303" } google-oauth-client = { group = "com.google.oauth-client", name = "google-oauth-client", version = "1.36.0" } diff --git a/model-server-openapi/specifications/model-server-v2.yaml b/model-server-openapi/specifications/model-server-v2.yaml index 526cd94f83..2f408a91e2 100644 --- a/model-server-openapi/specifications/model-server-v2.yaml +++ b/model-server-openapi/specifications/model-server-v2.yaml @@ -140,15 +140,36 @@ paths: required: true schema: type: string + x-modelix-media-type-handlers: + - v1: + - 'application/x.modelix.branch+json;version=1' + - delta: + - 'application/x-modelix-objects-v2' + - 'application/x-modelix-objects' + - 'application/json' + - 'text/plain' + - '*/*' responses: "404": $ref: '#/components/responses/404' "200": - $ref: '#/components/responses/versionDelta' -# content: -# '*/*': -# schema: -# $ref: "#/components/schemas/VersionDelta" + description: "Information about a branch for content type `application/x.modelix.branch+json;version=*'. Else all model data of the branch in version delta format." + content: + 'application/x.modelix.branch+json;version=1': + schema: + $ref: "#/components/schemas/BranchV1" + 'application/x-modelix-objects-v2': + schema: + type: string + 'application/x-modelix-objects': + schema: + type: string + 'application/json': + schema: + type: object + 'text/plain': + schema: + type: string default: $ref: '#/components/responses/GeneralError' post: @@ -538,3 +559,15 @@ components: type: string value2: type: string + BranchV1: + x-modelix-media-type: 'application/x.modelix.branch+json;version=1' + type: object + properties: + name: + type: string + current_hash: + type: string + example: 7fQeo*xrdfZuHZtaKhbp0OosarV5tVR8N3pW8JPkl7ZE + required: + - name + - current_hash diff --git a/model-server/build.gradle.kts b/model-server/build.gradle.kts index 583a492065..318b3ab661 100644 --- a/model-server/build.gradle.kts +++ b/model-server/build.gradle.kts @@ -66,6 +66,8 @@ dependencies { testImplementation(libs.bundles.apache.cxf) testImplementation(libs.junit) + testImplementation(libs.kotest.assertions.coreJvm) + testImplementation(libs.kotest.assertions.ktor) testImplementation(libs.cucumber.java) testImplementation(libs.ktor.server.test.host) testImplementation(libs.kotlin.coroutines.test) diff --git a/model-server/src/main/kotlin/org/modelix/model/server/Main.kt b/model-server/src/main/kotlin/org/modelix/model/server/Main.kt index d68632916b..eaf59e4042 100644 --- a/model-server/src/main/kotlin/org/modelix/model/server/Main.kt +++ b/model-server/src/main/kotlin/org/modelix/model/server/Main.kt @@ -55,6 +55,7 @@ import kotlinx.serialization.json.Json import org.apache.commons.io.FileUtils import org.apache.ignite.Ignition import org.modelix.api.v1.Problem +import org.modelix.api.v2.Paths.registerJsonTypes import org.modelix.authorization.KeycloakUtils import org.modelix.authorization.NoPermissionException import org.modelix.authorization.NotLoggedInException @@ -203,6 +204,7 @@ object Main { } install(ContentNegotiation) { json() + registerJsonTypes() } install(CORS) { anyHost() diff --git a/model-server/src/main/kotlin/org/modelix/model/server/handlers/ModelReplicationServer.kt b/model-server/src/main/kotlin/org/modelix/model/server/handlers/ModelReplicationServer.kt index a656561822..3d15d19180 100644 --- a/model-server/src/main/kotlin/org/modelix/model/server/handlers/ModelReplicationServer.kt +++ b/model-server/src/main/kotlin/org/modelix/model/server/handlers/ModelReplicationServer.kt @@ -39,6 +39,7 @@ import kotlinx.coroutines.flow.flow import kotlinx.coroutines.flow.onEmpty import kotlinx.coroutines.flow.withIndex import kotlinx.coroutines.withContext +import org.modelix.api.v2.BranchV1 import org.modelix.api.v2.DefaultApi import org.modelix.authorization.getUserName import org.modelix.model.InMemoryModels @@ -94,7 +95,9 @@ class ModelReplicationServer( } } - private fun repositoryId(paramValue: String?) = RepositoryId(checkNotNull(paramValue) { "Parameter 'repository' not available" }) + private fun repositoryId(paramValue: String?) = + RepositoryId(checkNotNull(paramValue) { "Parameter 'repository' not available" }) + private suspend fun runWithRepository(repository: String, body: suspend () -> R): R { return repositoriesManager.runWithRepository(repositoryId(repository), body) } @@ -107,7 +110,7 @@ class ModelReplicationServer( call.respondText(repositoriesManager.getBranchNames(repositoryId(repository)).joinToString("\n")) } - override suspend fun PipelineContext.getRepositoryBranch( + override suspend fun PipelineContext.getRepositoryBranchDelta( repository: String, branch: String, lastKnown: String?, @@ -119,6 +122,18 @@ class ModelReplicationServer( } } + override suspend fun PipelineContext.getRepositoryBranchV1( + repository: String, + branch: String, + lastKnown: String?, + ) { + runWithRepository(repository) { + val branchRef = repositoryId(repository).getBranchReference(branch) + val versionHash = repositoriesManager.getVersionHash(branchRef) ?: throw BranchNotFoundException(branchRef) + call.respond(BranchV1(branch, versionHash)) + } + } + override suspend fun PipelineContext.deleteRepositoryBranch( repository: String, branch: String, diff --git a/model-server/src/main/resources/openapi/templates/Paths.kt.mustache b/model-server/src/main/resources/openapi/templates/Paths.kt.mustache index 11a429ea45..7e05b4e43b 100644 --- a/model-server/src/main/resources/openapi/templates/Paths.kt.mustache +++ b/model-server/src/main/resources/openapi/templates/Paths.kt.mustache @@ -3,6 +3,10 @@ package {{packageName}} import io.ktor.resources.* import kotlinx.serialization.* +import io.ktor.http.ContentType +import io.ktor.serialization.kotlinx.KotlinxSerializationConverter +import io.ktor.serialization.kotlinx.json.DefaultJson +import io.ktor.server.plugins.contentnegotiation.ContentNegotiationConfig {{#imports}}import {{import}} {{/imports}} @@ -26,5 +30,22 @@ object Paths { {{/operation}} {{/operations}} {{/apis}} + + /** + * Registers all models from /components/schemas with an x-modelix-media-type vendor extension to be serializable + * as JSON for that media type. + */ + fun ContentNegotiationConfig.registerJsonTypes() { + {{#models}} + {{#model}} + {{#vendorExtensions}} + {{#x-modelix-media-type}} + register(ContentType.parse("{{{.}}}"), KotlinxSerializationConverter(DefaultJson)) + {{/x-modelix-media-type}} + {{/vendorExtensions}} + {{/model}} + {{/models}} + } + } {{/apiInfo}} diff --git a/model-server/src/main/resources/openapi/templates/api.mustache b/model-server/src/main/resources/openapi/templates/api.mustache index bca88a6ea3..4d47a3c227 100644 --- a/model-server/src/main/resources/openapi/templates/api.mustache +++ b/model-server/src/main/resources/openapi/templates/api.mustache @@ -3,6 +3,7 @@ package {{apiPackage}} import io.ktor.http.* import io.ktor.server.application.* +import io.ktor.server.request.* import io.ktor.server.response.* {{#featureResources}} import {{packageName}}.Paths @@ -22,6 +23,11 @@ import io.ktor.util.pipeline.PipelineContext abstract class {{classname}} { {{#operations}} {{#operation}} + + {{#vendorExtensions}} + {{#x-modelix-media-type-handlers}} + {{#entrySet}} + /**{{#summary}} * {{.}}{{/summary}} * @@ -29,14 +35,38 @@ abstract class {{classname}} { * * {{httpMethod}} {{path}} * - {{#allParams}}* @param {{paramName}} {{description}} {{^required}}(optional{{#defaultValue}}, default to {{{.}}}{{/defaultValue}}){{/required}} - {{/allParams}} + {{#allParams}} + * @param {{paramName}} {{description}} {{^required}}(optional{{#defaultValue}}, default to {{{.}}}{{/defaultValue}}){{/required}} + {{/allParams}} + */ + {{#isDeprecated}} + @Deprecated("deprecated flag is set in the OpenAPI specification") + {{/isDeprecated}} + abstract suspend fun PipelineContext.{{operationId}}{{#lambda.titlecase}}{{key}}{{/lambda.titlecase}}({{#allParams}}{{paramName}}: {{{dataType}}}{{^required}}?{{/required}}{{#required}}{{#isNullable}}?{{/isNullable}}{{/required}}{{^-last}}, {{/-last}}{{/allParams}}) + + {{/entrySet}} + {{/x-modelix-media-type-handlers}} + {{^x-modelix-media-type-handlers}} + + /**{{#summary}} + * {{.}}{{/summary}} + * + * {{unescapedNotes}} + * + * {{httpMethod}} {{path}} + * + {{#allParams}} + * @param {{paramName}} {{description}} {{^required}}(optional{{#defaultValue}}, default to {{{.}}}{{/defaultValue}}){{/required}} + {{/allParams}} */ {{#isDeprecated}} @Deprecated("deprecated flag is set in the OpenAPI specification") {{/isDeprecated}} abstract suspend fun PipelineContext.{{operationId}}({{#allParams}}{{paramName}}: {{{dataType}}}{{^required}}?{{/required}}{{#required}}{{#isNullable}}?{{/isNullable}}{{/required}}{{^-last}}, {{/-last}}{{/allParams}}) + {{/x-modelix-media-type-handlers}} + {{/vendorExtensions}} + {{/operation}} {{/operations}} @@ -55,9 +85,30 @@ abstract class {{classname}} { {{#operations}} {{#operation}} protected open fun Route.install_{{operationId}}() { + {{#vendorExtensions}} + {{#x-modelix-media-type-handlers}} + {{#entrySet}} + + accept( + {{#value}} + ContentType.parse("{{{.}}}"), + {{/value}} + ) { + {{#lambda.lowercase}}{{httpMethod}}{{/lambda.lowercase}} { parameters -> + {{operationId}}{{#lambda.titlecase}}{{key}}{{/lambda.titlecase}}({{#allParams}}parameters.{{paramName}}{{^-last}}, {{/-last}}{{/allParams}}) + } + } + + {{/entrySet}} + {{/x-modelix-media-type-handlers}} + {{^x-modelix-media-type-handlers}} + {{#lambda.lowercase}}{{httpMethod}}{{/lambda.lowercase}} { parameters -> - {{#lambda.indented_8}}{{operationId}}({{#allParams}}parameters.{{paramName}}{{^-last}}, {{/-last}}{{/allParams}}){{/lambda.indented_8}} + {{operationId}}({{#allParams}}parameters.{{paramName}}{{^-last}}, {{/-last}}{{/allParams}}) } + + {{/x-modelix-media-type-handlers}} + {{/vendorExtensions}} } {{/operation}} {{/operations}} diff --git a/model-server/src/test/kotlin/org/modelix/model/server/ModelServerTestUtil.kt b/model-server/src/test/kotlin/org/modelix/model/server/ModelServerTestUtil.kt index 6e9e91e74f..86fde1494f 100644 --- a/model-server/src/test/kotlin/org/modelix/model/server/ModelServerTestUtil.kt +++ b/model-server/src/test/kotlin/org/modelix/model/server/ModelServerTestUtil.kt @@ -27,6 +27,7 @@ import io.ktor.server.routing.IgnoreTrailingSlash import io.ktor.server.testing.ApplicationTestBuilder import io.ktor.server.websocket.WebSockets import kotlinx.coroutines.runBlocking +import org.modelix.api.v2.Paths.registerJsonTypes import org.modelix.authorization.installAuthentication import org.modelix.model.client2.ModelClientV2 import org.modelix.model.server.Main.installStatusPages @@ -38,7 +39,10 @@ suspend fun ApplicationTestBuilder.createModelClient(): ModelClientV2 { fun Application.installDefaultServerPlugins() { install(WebSockets) - install(ContentNegotiation) { json() } + install(ContentNegotiation) { + json() + registerJsonTypes() + } install(Resources) install(IgnoreTrailingSlash) installStatusPages() diff --git a/model-server/src/test/kotlin/org/modelix/model/server/handlers/ModelReplicationServerTest.kt b/model-server/src/test/kotlin/org/modelix/model/server/handlers/ModelReplicationServerTest.kt index c6e67882bd..9844c83085 100644 --- a/model-server/src/test/kotlin/org/modelix/model/server/handlers/ModelReplicationServerTest.kt +++ b/model-server/src/test/kotlin/org/modelix/model/server/handlers/ModelReplicationServerTest.kt @@ -15,11 +15,17 @@ package org.modelix.model.server.handlers +import com.google.api.client.http.HttpStatusCodes +import io.kotest.assertions.ktor.client.shouldHaveContentType +import io.kotest.assertions.ktor.client.shouldHaveStatus +import io.kotest.matchers.shouldBe import io.ktor.client.HttpClient import io.ktor.client.call.body import io.ktor.client.engine.cio.CIO import io.ktor.client.plugins.HttpTimeout +import io.ktor.client.plugins.contentnegotiation.ContentNegotiation import io.ktor.client.plugins.defaultRequest +import io.ktor.client.request.accept import io.ktor.client.request.delete import io.ktor.client.request.get import io.ktor.client.statement.bodyAsText @@ -29,6 +35,9 @@ import io.ktor.http.HttpStatusCode import io.ktor.http.appendPathSegments import io.ktor.http.contentType import io.ktor.http.takeFrom +import io.ktor.http.withCharset +import io.ktor.serialization.kotlinx.KotlinxSerializationConverter +import io.ktor.serialization.kotlinx.json.DefaultJson import io.ktor.serialization.kotlinx.json.json import io.ktor.server.application.Application import io.ktor.server.netty.NettyApplicationEngine @@ -42,6 +51,7 @@ import kotlinx.coroutines.flow.flow import kotlinx.coroutines.test.runTest import kotlinx.coroutines.withTimeout import org.modelix.api.v1.Problem +import org.modelix.api.v2.BranchV1 import org.modelix.authorization.installAuthentication import org.modelix.model.InMemoryModels import org.modelix.model.api.IConceptReference @@ -51,6 +61,7 @@ import org.modelix.model.client2.runWrite import org.modelix.model.client2.useVersionStreamFormat import org.modelix.model.lazy.RepositoryId import org.modelix.model.server.api.v2.VersionDeltaStream +import org.modelix.model.server.api.v2.VersionDeltaStreamV2 import org.modelix.model.server.installDefaultServerPlugins import org.modelix.model.server.runWithNettyServer import org.modelix.model.server.store.InMemoryStoreClient @@ -222,7 +233,14 @@ class ModelReplicationServerTest { val repositoryId = RepositoryId("repo1") val branchRef = repositoryId.getBranchReference() - runWithTestModelServer(Fixture(storeClient, modelClient, faultyRepositoriesManager, modelReplicationServer)) { _, _ -> + runWithTestModelServer( + Fixture( + storeClient, + modelClient, + faultyRepositoriesManager, + modelReplicationServer, + ), + ) { _, _ -> repositoriesManager.createRepository(repositoryId, null) // Act @@ -365,4 +383,53 @@ class ModelReplicationServerTest { ) } } + + @Test + fun `getRepositoryBranch responds with version delta v2 if requested`() { + val repositoryId = RepositoryId("repo1") + + runWithTestModelServer { _, fixture -> + fixture.repositoriesManager.createRepository(repositoryId, null) + + val response = client.get { + url { + appendPathSegments("v2", "repositories", repositoryId.id, "branches", "master") + } + accept(VersionDeltaStreamV2.CONTENT_TYPE) + } + + response shouldHaveStatus HttpStatusCodes.STATUS_CODE_OK + response.shouldHaveContentType(VersionDeltaStreamV2.CONTENT_TYPE) + } + } + + @Test + fun `getRepositoryBranch responds with v1 branch if requested`() { + val repositoryId = RepositoryId("repo1") + val branchV1ContentType = + ContentType.parse("application/x.modelix.branch+json;version=1").withCharset(Charsets.UTF_8) + + runWithTestModelServer { _, fixture -> + fixture.repositoriesManager.createRepository(repositoryId, null) + val client = createClient { + install(ContentNegotiation) { + json() + register(branchV1ContentType, KotlinxSerializationConverter(DefaultJson)) + } + } + + val response = client.get { + url { + appendPathSegments("v2", "repositories", repositoryId.id, "branches", "master") + } + accept(branchV1ContentType) + } + + response.shouldHaveContentType(branchV1ContentType) + response.body() shouldBe BranchV1( + "master", + fixture.repositoriesManager.pollVersionHash(repositoryId.getBranchReference("master"), null), + ) + } + } }