diff --git a/.gitignore b/.gitignore index 3123bc40..3a33ab27 100644 --- a/.gitignore +++ b/.gitignore @@ -44,3 +44,5 @@ build/ ### DEV _tmp/ .repository/ + +.console.sh diff --git a/README.md b/README.md index c033e35c..af062061 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,18 @@ # avro-kotlin Avro Kotlin provides a type- and null-safe type-system that encapsulates the apache-java avro-types and makes them -easily accessible from kotlin. +easily accessible from kotlin. The mission is to become your one-stop lib when dealing with avro and kotlin. [![stable](https://img.shields.io/badge/lifecycle-STABLE-green.svg)](https://github.com/holisticon#open-source-lifecycle) +[![Maven Central](https://maven-badges.herokuapp.com/maven-central/io.toolisticon.kotlin.avro/avro-kotlin-bom/badge.svg)](https://maven-badges.herokuapp.com/maven-central/io.toolisticon.kotlin.avro/avro-kotlin-bom) +[![Kotlin](https://img.shields.io/badge/kotlin-2.0.0-blue.svg?logo=kotlin)](http://kotlinlang.org) +[![Kotlinx serialization](https://img.shields.io/badge/kotlinx--serialization-1.7.1-blue?logo=kotlin)](https://github.com/Kotlin/kotlinx.serialization) +[![Avro spec](https://img.shields.io/badge/avro%20spec-1.11.3-blue.svg?logo=apache)](https://avro.apache.org/docs/1.11.3/specification/) [![Build Status](https://github.com/toolisticon/avro-kotlin/workflows/Development%20branches/badge.svg)](https://github.com/toolisticon/avro-kotlin/actions) [![sponsored](https://img.shields.io/badge/sponsoredBy-Holisticon-RED.svg)](https://holisticon.de/) -[![Maven Central](https://maven-badges.herokuapp.com/maven-central/io.toolisticon.kotlin.avro/avro-kotlin-bom/badge.svg)](https://maven-badges.herokuapp.com/maven-central/io.toolisticon.kotlin.avro/avro-kotlin-bom) - -Since Version 1.11.4.2 we use a multi module build. The GroupId changed, best use the provided BOM: +> [!NOTE] +> Since Version 1.11.4.2 we use a multi module build. The GroupId changed, best use the provided BOM: ```xml io.toolisticon.kotlin.avro diff --git a/_examples/java-example/pom.xml b/_examples/java-example/pom.xml index a86a3c78..e0fdb7d4 100644 --- a/_examples/java-example/pom.xml +++ b/_examples/java-example/pom.xml @@ -5,7 +5,7 @@ io.toolisticon.kotlin.avro._ avro-kotlin-examples-root - 1.11.4.4 + 1.11.5.0 io.toolisticon.kotlin.avro.examples diff --git a/_examples/java-example/src/test/java/io/toolisticon/kotlin/avro/example/java/BankAccountCreatedTest.java b/_examples/java-example/src/test/java/io/toolisticon/kotlin/avro/example/java/BankAccountCreatedTest.java index aede51bc..7285259f 100644 --- a/_examples/java-example/src/test/java/io/toolisticon/kotlin/avro/example/java/BankAccountCreatedTest.java +++ b/_examples/java-example/src/test/java/io/toolisticon/kotlin/avro/example/java/BankAccountCreatedTest.java @@ -1,12 +1,12 @@ package io.toolisticon.kotlin.avro.example.java; import io.toolisticon.example.bank.BankAccountCreated; +import io.toolisticon.kotlin.avro.repository.AvroSchemaResolverMap; import org.javamoney.moneta.Money; import org.junit.jupiter.api.Test; import java.util.UUID; -import static io.toolisticon.kotlin.avro.AvroKotlin.avroSchemaResolver; import static io.toolisticon.kotlin.avro.codec.SpecificRecordCodec.specificRecordSingleObjectDecoder; import static io.toolisticon.kotlin.avro.codec.SpecificRecordCodec.specificRecordSingleObjectEncoder; import static org.assertj.core.api.Assertions.assertThat; @@ -21,7 +21,7 @@ void encodeAndDecodeEventWithMoneyLogicalType() { .setCustomerId("1") .setInitialBalance(Money.of(100.123456, "EUR")) .build(); - final var resolver = avroSchemaResolver(BankAccountCreated.getClassSchema()); + final var resolver = new AvroSchemaResolverMap(BankAccountCreated.getClassSchema()); final var encoded = specificRecordSingleObjectEncoder().encode(bankAccountCreated); diff --git a/_examples/kotlin-example/pom.xml b/_examples/kotlin-example/pom.xml index 83dc64f2..12584990 100644 --- a/_examples/kotlin-example/pom.xml +++ b/_examples/kotlin-example/pom.xml @@ -5,7 +5,7 @@ io.toolisticon.kotlin.avro._ avro-kotlin-examples-root - 1.11.4.4 + 1.11.5.0 io.toolisticon.kotlin.avro.examples diff --git a/_examples/kotlin-example/src/main/kotlin/BankAccountCreatedData.kt b/_examples/kotlin-example/src/main/kotlin/BankAccountCreatedData.kt index 0f506a97..bd411294 100644 --- a/_examples/kotlin-example/src/main/kotlin/BankAccountCreatedData.kt +++ b/_examples/kotlin-example/src/main/kotlin/BankAccountCreatedData.kt @@ -1,18 +1,16 @@ package io.toolisticon.kotlin.avro.example -import com.github.avrokotlin.avro4k.AvroName -import com.github.avrokotlin.avro4k.AvroNamespace import com.github.avrokotlin.avro4k.serializer.UUIDSerializer import io.toolisticon.kotlin.avro.example.customerid.CustomerId import io.toolisticon.kotlin.avro.example.customerid.CustomerIdLogicalType import io.toolisticon.kotlin.avro.example.money.MoneyLogicalType.MoneySerializer +import kotlinx.serialization.SerialName import kotlinx.serialization.Serializable import org.javamoney.moneta.Money import java.util.* @Serializable -@AvroName("BankAccountCreated") -@AvroNamespace("io.toolisticon.bank") +@SerialName("io.toolisticon.bank.BankAccountCreated") data class BankAccountCreatedData( @Serializable(with = UUIDSerializer::class) diff --git a/_examples/kotlin-example/src/test/kotlin/BankAccountCreatedDataTest.kt b/_examples/kotlin-example/src/test/kotlin/BankAccountCreatedDataTest.kt index 1cc73b91..e0da5cc9 100644 --- a/_examples/kotlin-example/src/test/kotlin/BankAccountCreatedDataTest.kt +++ b/_examples/kotlin-example/src/test/kotlin/BankAccountCreatedDataTest.kt @@ -1,9 +1,9 @@ package io.toolisticon.kotlin.avro.example +import io.toolisticon.kotlin.avro.AvroKotlin import io.toolisticon.kotlin.avro.codec.GenericRecordCodec import io.toolisticon.kotlin.avro.example.customerid.CustomerId import io.toolisticon.kotlin.avro.example.money.MoneyLogicalType -import io.toolisticon.kotlin.avro.repository.avroSchemaResolver import io.toolisticon.kotlin.avro.serialization.AvroKotlinSerialization import io.toolisticon.kotlin.avro.value.CanonicalName.Companion.toCanonicalName import io.toolisticon.kotlin.avro.value.Name.Companion.toName @@ -17,7 +17,6 @@ internal class BankAccountCreatedDataTest { @Test fun `show schema`() { val schema = KotlinExample.avro.schema(BankAccountCreatedData::class) - println(schema) assertThat(schema.canonicalName).isEqualTo("io.toolisticon.bank.BankAccountCreated".toCanonicalName()) assertThat(schema.fields).hasSize(3) @@ -49,7 +48,7 @@ internal class BankAccountCreatedDataTest { val customerId = CustomerId.random() val event = BankAccountCreatedData(accountId, customerId, amount) - val resolver = avroSchemaResolver(KotlinExample.avro.schema(BankAccountCreatedData::class)) + val resolver = AvroKotlin.avroSchemaResolver(KotlinExample.avro.schema(BankAccountCreatedData::class)) val record = KotlinExample.avro.toRecord(event) diff --git a/_examples/kotlin-example/src/test/kotlin/JavaKotlinInterOpTest.kt b/_examples/kotlin-example/src/test/kotlin/JavaKotlinInterOpTest.kt index d4cc47fc..81008b19 100644 --- a/_examples/kotlin-example/src/test/kotlin/JavaKotlinInterOpTest.kt +++ b/_examples/kotlin-example/src/test/kotlin/JavaKotlinInterOpTest.kt @@ -1,10 +1,10 @@ package io.toolisticon.kotlin.avro.example import io.toolisticon.example.bank.BankAccountCreated +import io.toolisticon.kotlin.avro.AvroKotlin.avroSchemaResolver import io.toolisticon.kotlin.avro.codec.GenericRecordCodec import io.toolisticon.kotlin.avro.example.customerid.CustomerId import io.toolisticon.kotlin.avro.model.wrapper.AvroSchema -import io.toolisticon.kotlin.avro.repository.avroSchemaResolver import io.toolisticon.kotlin.avro.serialization.AvroKotlinSerialization import io.toolisticon.kotlin.avro.value.SingleObjectEncodedBytes import org.assertj.core.api.Assertions.assertThat @@ -54,6 +54,5 @@ internal class JavaKotlinInterOpTest { assertThat(decoded.accountId).isEqualTo(orig.accountId) assertThat(decoded.customerId).isEqualTo(orig.customerId) assertThat(decoded.initialBalance).isEqualTo(orig.initialBalance) - } } diff --git a/_examples/logical-type-customer-id/pom.xml b/_examples/logical-type-customer-id/pom.xml index 2310e796..6dfc97e3 100644 --- a/_examples/logical-type-customer-id/pom.xml +++ b/_examples/logical-type-customer-id/pom.xml @@ -5,7 +5,7 @@ io.toolisticon.kotlin.avro._ avro-kotlin-examples-root - 1.11.4.4 + 1.11.5.0 io.toolisticon.kotlin.avro.examples diff --git a/_examples/logical-type-money/pom.xml b/_examples/logical-type-money/pom.xml index be3bc3fd..b3faa0e5 100644 --- a/_examples/logical-type-money/pom.xml +++ b/_examples/logical-type-money/pom.xml @@ -5,7 +5,7 @@ io.toolisticon.kotlin.avro._ avro-kotlin-examples-root - 1.11.4.4 + 1.11.5.0 io.toolisticon.kotlin.avro.examples diff --git a/_examples/logical-type-money/src/test/kotlin/.gitkeep b/_examples/logical-type-money/src/test/kotlin/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/_examples/pom.xml b/_examples/pom.xml index ac3b288c..b555962e 100644 --- a/_examples/pom.xml +++ b/_examples/pom.xml @@ -5,7 +5,7 @@ io.toolisticon.kotlin.avro._ avro-kotlin-root - 1.11.4.4 + 1.11.5.0 avro-kotlin-examples-root diff --git a/_mvn/bom/pom.xml b/_mvn/bom/pom.xml index fa8e7fe2..5ec48a5b 100644 --- a/_mvn/bom/pom.xml +++ b/_mvn/bom/pom.xml @@ -5,7 +5,7 @@ io.toolisticon.kotlin.avro._ avro-kotlin-root - 1.11.4.4 + 1.11.5.0 ../../pom.xml diff --git a/_mvn/coverage-aggregator/pom.xml b/_mvn/coverage-aggregator/pom.xml index de958315..9bf7c07f 100644 --- a/_mvn/coverage-aggregator/pom.xml +++ b/_mvn/coverage-aggregator/pom.xml @@ -5,7 +5,7 @@ io.toolisticon.kotlin.avro._ avro-kotlin-root - 1.11.4.4 + 1.11.5.0 ../../pom.xml diff --git a/_mvn/parent/pom.xml b/_mvn/parent/pom.xml index 5e485401..2b693926 100644 --- a/_mvn/parent/pom.xml +++ b/_mvn/parent/pom.xml @@ -5,7 +5,7 @@ io.toolisticon.kotlin.avro._ avro-kotlin-root - 1.11.4.4 + 1.11.5.0 ../../pom.xml diff --git a/_test/pom.xml b/_test/pom.xml index 09b51710..847e4fe6 100644 --- a/_test/pom.xml +++ b/_test/pom.xml @@ -5,7 +5,7 @@ io.toolisticon.kotlin.avro._ avro-kotlin-parent - 1.11.4.4 + 1.11.5.0 ../_mvn/parent/pom.xml diff --git a/avro-kotlin-serialization/pom.xml b/avro-kotlin-serialization/pom.xml index da28930d..6bc690f2 100644 --- a/avro-kotlin-serialization/pom.xml +++ b/avro-kotlin-serialization/pom.xml @@ -5,7 +5,7 @@ io.toolisticon.kotlin.avro._ avro-kotlin-parent - 1.11.4.4 + 1.11.5.0 ../_mvn/parent/pom.xml diff --git a/avro-kotlin-serialization/src/main/kotlin/AvroKotlinSerialization.kt b/avro-kotlin-serialization/src/main/kotlin/AvroKotlinSerialization.kt index 7a47d896..2c966780 100644 --- a/avro-kotlin-serialization/src/main/kotlin/AvroKotlinSerialization.kt +++ b/avro-kotlin-serialization/src/main/kotlin/AvroKotlinSerialization.kt @@ -1,75 +1,147 @@ package io.toolisticon.kotlin.avro.serialization -import com.github.avrokotlin.avro4k.Avro +import com.github.avrokotlin.avro4k.* import io.toolisticon.kotlin.avro.AvroKotlin +import io.toolisticon.kotlin.avro.codec.AvroCodec import io.toolisticon.kotlin.avro.codec.GenericRecordCodec import io.toolisticon.kotlin.avro.model.wrapper.AvroSchema -import io.toolisticon.kotlin.avro.model.wrapper.AvroSchemaChecks.compatibleToReadFrom import io.toolisticon.kotlin.avro.repository.AvroSchemaResolver +import io.toolisticon.kotlin.avro.repository.AvroSchemaResolverMutableMap +import io.toolisticon.kotlin.avro.serialization.avro4k.avro4k import io.toolisticon.kotlin.avro.serialization.spi.AvroSerializationModuleFactoryServiceLoader import io.toolisticon.kotlin.avro.serialization.spi.SerializerModuleKtx.reduce +import io.toolisticon.kotlin.avro.value.AvroSchemaCompatibilityMap import io.toolisticon.kotlin.avro.value.SingleObjectEncodedBytes +import kotlinx.serialization.ExperimentalSerializationApi import kotlinx.serialization.KSerializer import kotlinx.serialization.modules.SerializersModule +import kotlinx.serialization.serializer import mu.KLogging import org.apache.avro.generic.GenericData import org.apache.avro.generic.GenericRecord +import java.lang.Runtime.Version import java.util.concurrent.ConcurrentHashMap import kotlin.reflect.KClass +import kotlin.reflect.full.createType +@OptIn(ExperimentalSerializationApi::class) class AvroKotlinSerialization( - private val avro4k: Avro -) { + val avro4k: Avro, + private val schemaResolver: AvroSchemaResolverMutableMap = AvroSchemaResolverMutableMap.EMPTY, + private val genericData: GenericData = AvroKotlin.genericData +) : AvroSchemaResolver by schemaResolver { companion object : KLogging() { fun configure(vararg serializersModules: SerializersModule): AvroKotlinSerialization { return AvroKotlinSerialization( - Avro( + Avro { serializersModule = serializersModules.toList().reduce() - ) + } ) } } + init { + /** + * We _need_ `kotlinx.serialization >= 1.7`. spring boot provides an outdated version (1.6.3). + * This is a pita to resolve. This check makes sure, any misconfigurations are found on app-start. + */ + check(Version.parse(KSerializer::class.java.`package`.implementationVersion) >= Version.parse("1.7")) { "avro4k uses features that required kotlinx.serialization version >= 1.7.0. Make sure to include the correct versions, especially when you use spring-boot." } + } + + val avro4kSingleObject = AvroSingleObject(schemaRegistry = schemaResolver.avro4k(), avro = avro4k) + + val compatibilityCache = AvroSchemaCompatibilityMap() + private val kserializerCache = ConcurrentHashMap, KSerializer<*>>() private val schemaCache = ConcurrentHashMap, AvroSchema>() constructor() : this( - Avro(AvroSerializationModuleFactoryServiceLoader()) + Avro { serializersModule = AvroSerializationModuleFactoryServiceLoader() } ) - fun serializer(type: KClass<*>) = kserializerCache.computeIfAbsent(type) { key -> - logger.trace { "add kserializer for $type." } - key.kserializer() + fun singleObjectEncoder(): AvroCodec.SingleObjectEncoder = AvroCodec.SingleObjectEncoder { data -> + @Suppress("UNCHECKED_CAST") + val serializer = serializer(data::class) as KSerializer + val writerSchema = schema(data::class) + + val bytes = avro4kSingleObject.encodeToByteArray(writerSchema.get(), serializer, data) + + SingleObjectEncodedBytes.of(bytes) } - fun schema(type: Class<*>): AvroSchema = schema(type.kotlin) + fun singleObjectDecoder(): AvroCodec.SingleObjectDecoder = AvroCodec.SingleObjectDecoder { bytes -> + val writerSchema = schemaResolver[bytes.fingerprint] + val klass = AvroKotlin.loadClassForSchema(writerSchema) - fun schema(type: KClass<*>): AvroSchema = schemaCache.computeIfAbsent(type) { key -> - logger.trace { "add schema for $type." } - AvroSchema(avro4k.schema(serializer(key))) + @Suppress("UNCHECKED_CAST") + avro4kSingleObject.decodeFromByteArray(serializer(klass), bytes.value) as T + } + + fun genericRecordEncoder(): AvroCodec.GenericRecordEncoder = AvroCodec.GenericRecordEncoder { data -> + @Suppress("UNCHECKED_CAST") + val serializer = serializer(data::class) as KSerializer + val writerSchema = schema(data::class) + + avro4k.encodeToGenericData(writerSchema.get(), serializer, data) as GenericRecord + } + + /** + * @param klass - optional. If we do know the klass already, we can pass it to avoid a second lookup. + */ + fun genericRecordDecoder(klass: KClass? = null) = AvroCodec.GenericRecordDecoder { record -> + val writerSchema = AvroSchema(record.schema) + val readerKlass: KClass = klass ?: AvroKotlin.loadClassForSchema(writerSchema) + + @Suppress("UNCHECKED_CAST") + val kserializer = serializer(readerKlass) as KSerializer + val readerSchema = schema(readerKlass) + val compatibility = compatibilityCache.compatibleToReadFrom(writerSchema, readerSchema) + + require(compatibility.isCompatible) { "Reader/writer schema are incompatible." } + + avro4k.decodeFromGenericData(writerSchema = writerSchema.get(), deserializer = kserializer, record) } @Suppress("UNCHECKED_CAST") - fun toRecord(data: T): GenericRecord { + fun toGenericRecord(data: T): GenericRecord { val kserializer = serializer(data::class) as KSerializer + val schema = avro4k.schema(kserializer) - return avro4k.toRecord(kserializer, data) + return avro4k.encodeToGenericData(schema, kserializer, data) as GenericRecord } - inline fun fromRecord(record: GenericRecord): T = fromRecord(record, T::class) + fun toSingleObjectEncoded(data: T): SingleObjectEncodedBytes = singleObjectEncoder().encode(data) - @Suppress("UNCHECKED_CAST") - fun fromRecord(record: GenericRecord, type: KClass): T { - val writerSchema = AvroSchema(record.schema) + /** + * @return kotlinx-serializer for given class. + */ + fun serializer(klass: KClass<*>) = kserializerCache.computeIfAbsent(klass) { key -> + + require(klass.isSerializable()) + + // TODO: if we use SpecificRecords, we could derive the schema from the class directly + logger.trace { "add kserializer for $key." } + + // TODO: createType takes a lot of optional args. We probably won't need them but at least we should check them. + val type = key.createType() + + avro4k.serializersModule.serializer(type) + } + + // TODO We would like to use the reified function from avro4k but we need to be able to dynamically load class by fqn + fun schema(type: KClass<*>): AvroSchema = schemaCache.computeIfAbsent(type) { key -> + logger.trace { "add schema for $type." } + AvroSchema(avro4k.schema(serializer(key))).also(this::registerSchema) + } - val kserializer = serializer(type) as KSerializer - val readerSchema = schema(type) + @Deprecated("use toGenericRecord instead", ReplaceWith("toGenericRecord(data")) + fun toRecord(data: T): GenericRecord = toGenericRecord(data) - // TODO nicer? - require(readerSchema.compatibleToReadFrom(writerSchema).result.incompatibilities.isEmpty()) { "Reader/writer schema are incompatible" } + inline fun fromRecord(record: GenericRecord): T = fromRecord(record, T::class) - return avro4k.fromRecord(kserializer, record) as T + fun fromRecord(record: GenericRecord, type: KClass): T { + return genericRecordDecoder(type).decode(record) } fun encodeSingleObject( @@ -108,4 +180,8 @@ class AvroKotlinSerialization( readerType = T::class ) + fun registerSchema(schema: AvroSchema): AvroKotlinSerialization = apply { schemaResolver + schema } + + fun cachedSerializerClasses() = kserializerCache.keys().toList().toSet() + fun cachedSchemaClasses() = schemaCache.keys().toList().toSet() } diff --git a/avro-kotlin-serialization/src/main/kotlin/_kserializer-reflection.kt b/avro-kotlin-serialization/src/main/kotlin/_kserializer-reflection.kt deleted file mode 100644 index d1a17a77..00000000 --- a/avro-kotlin-serialization/src/main/kotlin/_kserializer-reflection.kt +++ /dev/null @@ -1,29 +0,0 @@ -package io.toolisticon.kotlin.avro.serialization - -import kotlinx.serialization.KSerializer -import kotlinx.serialization.Serializable -import kotlin.reflect.KClass -import kotlin.reflect.full.companionObject -import kotlin.reflect.full.companionObjectInstance -import kotlin.reflect.full.functions - -/** - * Reflective access to [KSerializer<*>] for a given type. - * - * The type has to be a data class or enum and annotated with [Serializable]. - * - * @param type the class to access the serializer on - * @return kserializer of given type - * @throws IllegalArgumentException when type is not a valid [Serializable] type - */ -@Throws(IllegalArgumentException::class) -fun KClass<*>.kserializer(): KSerializer<*> { - require(this.isData) { "Type ${this.qualifiedName} is not a data class." } - require(this.annotations.any { it is Serializable }) { "Type ${this.qualifiedName} is not serializable." } - - val serializerFn = requireNotNull( - this.companionObject?.functions?.find { it.name == "serializer" } - ) { "Type ${this.qualifiedName} must have a Companion.serializer, as created by the serialization compiler plugin." } - - return serializerFn.call(this.companionObjectInstance) as KSerializer<*> -} diff --git a/avro-kotlin-serialization/src/main/kotlin/_reflection.kt b/avro-kotlin-serialization/src/main/kotlin/_reflection.kt new file mode 100644 index 00000000..73ee3def --- /dev/null +++ b/avro-kotlin-serialization/src/main/kotlin/_reflection.kt @@ -0,0 +1,43 @@ +package io.toolisticon.kotlin.avro.serialization + +import kotlinx.serialization.KSerializer +import kotlinx.serialization.Serializable +import org.apache.avro.specific.SpecificRecordBase +import kotlin.reflect.KClass +import kotlin.reflect.full.companionObject +import kotlin.reflect.full.companionObjectInstance +import kotlin.reflect.full.functions + +/** + * Reflective access to [KSerializer<*>] for a given type. + * + * The type has to be a data class or enum and annotated with [Serializable]. + * + * @return kserializer of given type + * @throws IllegalArgumentException when type is not a valid [Serializable] type + */ +@Throws(IllegalArgumentException::class) +@Deprecated("provided directly by kotlinx.serialization") +fun KClass<*>.kserializer(): KSerializer<*> { + require(this.isData) { "Type ${this.qualifiedName} is not a data class." } + require(this.isSerializable()) { "Type ${this.qualifiedName} is not serializable." } + + val serializerFn = + requireNotNull(this.companionObject?.functions?.find { it.name == "serializer" }) { "Type ${this.qualifiedName} must have a Companion.serializer, as created by the serialization compiler plugin." } + + return serializerFn.call(this.companionObjectInstance) as KSerializer<*> +} + +fun KClass<*>.isSerializable(): Boolean = this.annotations.any { it is Serializable } + +fun KClass<*>.isKotlinxDataClass(): Boolean { + // TODO: can this check be replaced by some convenience magic from kotlinx.serialization + return this.isData && this.isSerializable() +} + +fun KClass<*>.isKotlinxEnumClass(): Boolean { + // TODO: can this check be replaced by some convenience magic from kotlinx.serialization + return this.java.isEnum && this.isSerializable() +} + +fun KClass<*>.isGeneratedSpecificRecordBase(): Boolean = SpecificRecordBase::class.java.isAssignableFrom(this.java) diff --git a/avro-kotlin-serialization/src/main/kotlin/avro4k/Avro4kSchemaRegistry.kt b/avro-kotlin-serialization/src/main/kotlin/avro4k/Avro4kSchemaRegistry.kt new file mode 100644 index 00000000..6981645a --- /dev/null +++ b/avro-kotlin-serialization/src/main/kotlin/avro4k/Avro4kSchemaRegistry.kt @@ -0,0 +1,26 @@ +package io.toolisticon.kotlin.avro.serialization.avro4k + +import io.toolisticon.kotlin.avro.repository.AvroSchemaResolver +import org.apache.avro.Schema +import org.apache.avro.message.MissingSchemaException + +/** + * Avro4k provides [com.github.avrokotlin.avro4k.AvroSingleObject], which does require a `(Long) -> Schema?` function + * to resolve [org.apache.avro.Schema] from a store. + * + * This registry implements this requirement using our own [AvroSchemaResolver]. + */ +@JvmInline +value class Avro4kSchemaRegistry(private val schemaResolver: AvroSchemaResolver) : (Long) -> Schema?{ + + override fun invoke(fingerprint: Long): Schema? = try { + schemaResolver.findByFingerprint(fingerprint) + } catch (e: MissingSchemaException) { + null + } +} + +/** + * Extension fn to fluently crate a [Avro4kSchemaRegistry] from a [AvroSchemaResolver]. + */ +fun AvroSchemaResolver.avro4k(): Avro4kSchemaRegistry = Avro4kSchemaRegistry(this) diff --git a/avro-kotlin-serialization/src/main/kotlin/serializer/AvroLogicalTypeSerializer.kt b/avro-kotlin-serialization/src/main/kotlin/serializer/AvroLogicalTypeSerializer.kt index 68822e2b..2335cce1 100644 --- a/avro-kotlin-serialization/src/main/kotlin/serializer/AvroLogicalTypeSerializer.kt +++ b/avro-kotlin-serialization/src/main/kotlin/serializer/AvroLogicalTypeSerializer.kt @@ -1,28 +1,28 @@ package io.toolisticon.kotlin.avro.serialization.serializer import _ktx.StringKtx -import com.github.avrokotlin.avro4k.schema.AvroDescriptor -import com.github.avrokotlin.avro4k.schema.NamingStrategy +import com.github.avrokotlin.avro4k.AvroDecoder +import com.github.avrokotlin.avro4k.AvroEncoder import com.github.avrokotlin.avro4k.serializer.AvroSerializer +import com.github.avrokotlin.avro4k.serializer.SchemaSupplierContext import io.toolisticon.kotlin.avro.logical.AvroLogicalType import io.toolisticon.kotlin.avro.logical.conversion.AvroLogicalTypeConversion import kotlinx.serialization.ExperimentalSerializationApi import kotlinx.serialization.descriptors.PrimitiveKind -import kotlinx.serialization.descriptors.SerialDescriptor -import kotlinx.serialization.modules.SerializersModule import org.apache.avro.Schema @OptIn(ExperimentalSerializationApi::class) sealed class AvroLogicalTypeSerializer, JVM_TYPE : Any, CONVERTED_TYPE : Any>( val conversion: AvroLogicalTypeConversion, private val primitiveKind: PrimitiveKind -) : AvroSerializer() { +) : AvroSerializer(descriptorName = conversion.convertedType.qualifiedName!!) { + abstract override fun deserializeAvro(decoder: AvroDecoder): CONVERTED_TYPE - override val descriptor: SerialDescriptor = object : AvroDescriptor(type = conversion.convertedType, kind = primitiveKind) { - override fun schema(annos: List, serializersModule: SerializersModule, namingStrategy: NamingStrategy): Schema { - return conversion.logicalType.schema().get() - } + abstract override fun serializeAvro(encoder: AvroEncoder, value: CONVERTED_TYPE) + + override fun getSchema(context: SchemaSupplierContext): Schema { + return conversion.logicalType.schema().get() } override fun toString(): String = StringKtx.toString(this::class.java.simpleName) { diff --git a/avro-kotlin-serialization/src/main/kotlin/serializer/BooleanLogicalTypeSerializer.kt b/avro-kotlin-serialization/src/main/kotlin/serializer/BooleanLogicalTypeSerializer.kt index cb73064f..c9bae604 100644 --- a/avro-kotlin-serialization/src/main/kotlin/serializer/BooleanLogicalTypeSerializer.kt +++ b/avro-kotlin-serialization/src/main/kotlin/serializer/BooleanLogicalTypeSerializer.kt @@ -1,20 +1,26 @@ package io.toolisticon.kotlin.avro.serialization.serializer -import com.github.avrokotlin.avro4k.decoder.ExtendedDecoder -import com.github.avrokotlin.avro4k.encoder.ExtendedEncoder + +import com.github.avrokotlin.avro4k.AvroDecoder +import com.github.avrokotlin.avro4k.AvroEncoder import io.toolisticon.kotlin.avro.logical.BooleanLogicalType import io.toolisticon.kotlin.avro.logical.conversion.BooleanLogicalTypeConversion +import kotlinx.serialization.ExperimentalSerializationApi import kotlinx.serialization.descriptors.PrimitiveKind +import kotlinx.serialization.encoding.Decoder +import kotlinx.serialization.encoding.Encoder import org.apache.avro.Schema +@OptIn(ExperimentalSerializationApi::class) abstract class BooleanLogicalTypeSerializer( conversion: BooleanLogicalTypeConversion ) : AvroLogicalTypeSerializer( conversion = conversion, primitiveKind = PrimitiveKind.BOOLEAN ) { - override fun decodeAvroValue(schema: Schema, decoder: ExtendedDecoder): CONVERTED_TYPE { - val value = requireNotNull(decoder.decodeAny()) { "Can't deserialize null" } + + override fun deserializeAvro(decoder: AvroDecoder): CONVERTED_TYPE { + val value = requireNotNull(decoder.decodeValue()) { "Can't deserialize null" } @Suppress("UNCHECKED_CAST") return when(value::class) { conversion.convertedType -> value as CONVERTED_TYPE @@ -22,7 +28,7 @@ abstract class BooleanLogicalTypeSerializer( conversion: BytesLogicalTypeConversion ) : AvroLogicalTypeSerializer( conversion = conversion, primitiveKind = PrimitiveKind.BYTE ) { - override fun decodeAvroValue(schema: Schema, decoder: ExtendedDecoder): CONVERTED_TYPE { - val value = requireNotNull(decoder.decodeAny()) { "Can't deserialize null" } + + + override fun serializeAvro(encoder: AvroEncoder, value: CONVERTED_TYPE) { + val buffer = ByteBuffer.wrap(conversion.toAvro(value)) + encoder.encodeBytes(buffer) + } + + override fun deserializeAvro(decoder: AvroDecoder): CONVERTED_TYPE { + val value = requireNotNull(decoder.decodeValue()) { "Can't deserialize null" } @Suppress("UNCHECKED_CAST") - return when(value::class) { + return when (value::class) { conversion.convertedType -> value as CONVERTED_TYPE ByteArray::class -> conversion.fromAvro(value as ByteArray) else -> throw SerializationException("Can't deserialize $value") } } - - override fun encodeAvroValue(schema: Schema, encoder: ExtendedEncoder, obj: CONVERTED_TYPE) { - val buffer = ByteBuffer.wrap(conversion.toAvro(obj)) - return encoder.encodeByteArray(buffer) - } } diff --git a/avro-kotlin-serialization/src/main/kotlin/serializer/DoubleLogicalTypeSerializer.kt b/avro-kotlin-serialization/src/main/kotlin/serializer/DoubleLogicalTypeSerializer.kt index 372f09df..40ded1f5 100644 --- a/avro-kotlin-serialization/src/main/kotlin/serializer/DoubleLogicalTypeSerializer.kt +++ b/avro-kotlin-serialization/src/main/kotlin/serializer/DoubleLogicalTypeSerializer.kt @@ -1,28 +1,31 @@ package io.toolisticon.kotlin.avro.serialization.serializer -import com.github.avrokotlin.avro4k.decoder.ExtendedDecoder -import com.github.avrokotlin.avro4k.encoder.ExtendedEncoder +import com.github.avrokotlin.avro4k.AvroDecoder +import com.github.avrokotlin.avro4k.AvroEncoder import io.toolisticon.kotlin.avro.logical.DoubleLogicalType import io.toolisticon.kotlin.avro.logical.conversion.DoubleLogicalTypeConversion +import kotlinx.serialization.ExperimentalSerializationApi import kotlinx.serialization.descriptors.PrimitiveKind -import org.apache.avro.Schema +@OptIn(ExperimentalSerializationApi::class) abstract class DoubleLogicalTypeSerializer( conversion: DoubleLogicalTypeConversion ) : AvroLogicalTypeSerializer( conversion = conversion, primitiveKind = PrimitiveKind.DOUBLE ) { - override fun decodeAvroValue(schema: Schema, decoder: ExtendedDecoder): CONVERTED_TYPE { - val value = requireNotNull(decoder.decodeAny()) { "Can't deserialize null" } + + + override fun serializeAvro(encoder: AvroEncoder, value: CONVERTED_TYPE) { + encoder.encodeDouble(conversion.toAvro(value)) + } + + override fun deserializeAvro(decoder: AvroDecoder): CONVERTED_TYPE { + val value = requireNotNull(decoder.decodeValue()) { "Can't deserialize null" } @Suppress("UNCHECKED_CAST") - return when(value::class) { + return when (value::class) { conversion.convertedType -> value as CONVERTED_TYPE else -> conversion.fromAvro(decoder.decodeDouble()) } } - - override fun encodeAvroValue(schema: Schema, encoder: ExtendedEncoder, obj: CONVERTED_TYPE) { - return encoder.encodeDouble(conversion.toAvro(obj)) - } } diff --git a/avro-kotlin-serialization/src/main/kotlin/serializer/FloatLogicalTypeSerializer.kt b/avro-kotlin-serialization/src/main/kotlin/serializer/FloatLogicalTypeSerializer.kt index 95cfa0bb..3c9b9a22 100644 --- a/avro-kotlin-serialization/src/main/kotlin/serializer/FloatLogicalTypeSerializer.kt +++ b/avro-kotlin-serialization/src/main/kotlin/serializer/FloatLogicalTypeSerializer.kt @@ -1,28 +1,31 @@ package io.toolisticon.kotlin.avro.serialization.serializer -import com.github.avrokotlin.avro4k.decoder.ExtendedDecoder -import com.github.avrokotlin.avro4k.encoder.ExtendedEncoder +import com.github.avrokotlin.avro4k.AvroDecoder +import com.github.avrokotlin.avro4k.AvroEncoder import io.toolisticon.kotlin.avro.logical.FloatLogicalType import io.toolisticon.kotlin.avro.logical.conversion.FloatLogicalTypeConversion +import kotlinx.serialization.ExperimentalSerializationApi import kotlinx.serialization.descriptors.PrimitiveKind -import org.apache.avro.Schema +@OptIn(ExperimentalSerializationApi::class) abstract class FloatLogicalTypeSerializer( conversion: FloatLogicalTypeConversion ) : AvroLogicalTypeSerializer( conversion = conversion, primitiveKind = PrimitiveKind.FLOAT ) { - override fun decodeAvroValue(schema: Schema, decoder: ExtendedDecoder): CONVERTED_TYPE { - val value = requireNotNull(decoder.decodeAny()) { "Can't deserialize null" } + + + override fun serializeAvro(encoder: AvroEncoder, value: CONVERTED_TYPE) { + encoder.encodeFloat(conversion.toAvro(value)) + } + + override fun deserializeAvro(decoder: AvroDecoder): CONVERTED_TYPE { + val value = requireNotNull(decoder.decodeValue()) { "Can't deserialize null" } @Suppress("UNCHECKED_CAST") - return when(value::class) { + return when (value::class) { conversion.convertedType -> value as CONVERTED_TYPE else -> conversion.fromAvro(decoder.decodeFloat()) } } - - override fun encodeAvroValue(schema: Schema, encoder: ExtendedEncoder, obj: CONVERTED_TYPE) { - return encoder.encodeFloat(conversion.toAvro(obj)) - } } diff --git a/avro-kotlin-serialization/src/main/kotlin/serializer/IntLogicalTypeSerializer.kt b/avro-kotlin-serialization/src/main/kotlin/serializer/IntLogicalTypeSerializer.kt index 20741362..4b492868 100644 --- a/avro-kotlin-serialization/src/main/kotlin/serializer/IntLogicalTypeSerializer.kt +++ b/avro-kotlin-serialization/src/main/kotlin/serializer/IntLogicalTypeSerializer.kt @@ -1,28 +1,31 @@ package io.toolisticon.kotlin.avro.serialization.serializer -import com.github.avrokotlin.avro4k.decoder.ExtendedDecoder -import com.github.avrokotlin.avro4k.encoder.ExtendedEncoder +import com.github.avrokotlin.avro4k.AvroDecoder +import com.github.avrokotlin.avro4k.AvroEncoder import io.toolisticon.kotlin.avro.logical.IntLogicalType import io.toolisticon.kotlin.avro.logical.conversion.IntLogicalTypeConversion +import kotlinx.serialization.ExperimentalSerializationApi import kotlinx.serialization.descriptors.PrimitiveKind import org.apache.avro.Schema +@OptIn(ExperimentalSerializationApi::class) abstract class IntLogicalTypeSerializer( conversion: IntLogicalTypeConversion ) : AvroLogicalTypeSerializer( conversion = conversion, primitiveKind = PrimitiveKind.INT ) { - override fun decodeAvroValue(schema: Schema, decoder: ExtendedDecoder): CONVERTED_TYPE { - val value = requireNotNull(decoder.decodeAny()) { "Can't deserialize null" } + + override fun serializeAvro(encoder: AvroEncoder, value: CONVERTED_TYPE) { + encoder.encodeInt(conversion.toAvro(value)) + } + + override fun deserializeAvro(decoder: AvroDecoder): CONVERTED_TYPE { + val value = requireNotNull(decoder.decodeValue()) { "Can't deserialize null" } @Suppress("UNCHECKED_CAST") return when(value::class) { conversion.convertedType -> value as CONVERTED_TYPE else -> conversion.fromAvro(decoder.decodeInt()) } } - - override fun encodeAvroValue(schema: Schema, encoder: ExtendedEncoder, obj: CONVERTED_TYPE) { - return encoder.encodeInt(conversion.toAvro(obj)) - } } diff --git a/avro-kotlin-serialization/src/main/kotlin/serializer/LongLogicalTypeSerializer.kt b/avro-kotlin-serialization/src/main/kotlin/serializer/LongLogicalTypeSerializer.kt index 52966fb7..5ffbe8d8 100644 --- a/avro-kotlin-serialization/src/main/kotlin/serializer/LongLogicalTypeSerializer.kt +++ b/avro-kotlin-serialization/src/main/kotlin/serializer/LongLogicalTypeSerializer.kt @@ -1,28 +1,30 @@ package io.toolisticon.kotlin.avro.serialization.serializer -import com.github.avrokotlin.avro4k.decoder.ExtendedDecoder -import com.github.avrokotlin.avro4k.encoder.ExtendedEncoder +import com.github.avrokotlin.avro4k.AvroDecoder +import com.github.avrokotlin.avro4k.AvroEncoder import io.toolisticon.kotlin.avro.logical.LongLogicalType import io.toolisticon.kotlin.avro.logical.conversion.LongLogicalTypeConversion +import kotlinx.serialization.ExperimentalSerializationApi import kotlinx.serialization.descriptors.PrimitiveKind -import org.apache.avro.Schema +@OptIn(ExperimentalSerializationApi::class) abstract class LongLogicalTypeSerializer( conversion: LongLogicalTypeConversion ) : AvroLogicalTypeSerializer( conversion = conversion, primitiveKind = PrimitiveKind.LONG ) { - override fun decodeAvroValue(schema: Schema, decoder: ExtendedDecoder): CONVERTED_TYPE { - val value = requireNotNull(decoder.decodeAny()) { "Can't deserialize null" } + + override fun serializeAvro(encoder: AvroEncoder, value: CONVERTED_TYPE) { + encoder.encodeLong(conversion.toAvro(value)) + } + + override fun deserializeAvro(decoder: AvroDecoder): CONVERTED_TYPE { + val value = requireNotNull(decoder.decodeValue()) { "Can't deserialize null" } @Suppress("UNCHECKED_CAST") - return when(value::class) { + return when (value::class) { conversion.convertedType -> value as CONVERTED_TYPE else -> conversion.fromAvro(decoder.decodeLong()) } } - - override fun encodeAvroValue(schema: Schema, encoder: ExtendedEncoder, obj: CONVERTED_TYPE) { - return encoder.encodeLong(conversion.toAvro(obj)) - } } diff --git a/avro-kotlin-serialization/src/main/kotlin/serializer/StringLogicalTypeSerializer.kt b/avro-kotlin-serialization/src/main/kotlin/serializer/StringLogicalTypeSerializer.kt index 02e31edd..91e9b4c4 100644 --- a/avro-kotlin-serialization/src/main/kotlin/serializer/StringLogicalTypeSerializer.kt +++ b/avro-kotlin-serialization/src/main/kotlin/serializer/StringLogicalTypeSerializer.kt @@ -1,30 +1,30 @@ package io.toolisticon.kotlin.avro.serialization.serializer -import com.github.avrokotlin.avro4k.decoder.ExtendedDecoder -import com.github.avrokotlin.avro4k.encoder.ExtendedEncoder +import com.github.avrokotlin.avro4k.AvroDecoder +import com.github.avrokotlin.avro4k.AvroEncoder import io.toolisticon.kotlin.avro.logical.StringLogicalType import io.toolisticon.kotlin.avro.logical.conversion.StringLogicalTypeConversion -import kotlinx.serialization.SerializationException +import kotlinx.serialization.ExperimentalSerializationApi import kotlinx.serialization.descriptors.PrimitiveKind -import org.apache.avro.Schema -import org.apache.avro.util.Utf8 +@OptIn(ExperimentalSerializationApi::class) abstract class StringLogicalTypeSerializer( conversion: StringLogicalTypeConversion ) : AvroLogicalTypeSerializer( conversion = conversion, primitiveKind = PrimitiveKind.STRING ) { - override fun decodeAvroValue(schema: Schema, decoder: ExtendedDecoder): CONVERTED_TYPE { - val value = requireNotNull(decoder.decodeAny()) { "Can't deserialize null" } + + override fun serializeAvro(encoder: AvroEncoder, value: CONVERTED_TYPE) { + return encoder.encodeString(conversion.toAvro(value)) + } + + override fun deserializeAvro(decoder: AvroDecoder): CONVERTED_TYPE { + val value = requireNotNull(decoder.decodeValue()) { "Can't deserialize null" } @Suppress("UNCHECKED_CAST") - return when(value::class) { + return when (value::class) { conversion.convertedType -> value as CONVERTED_TYPE else -> conversion.fromAvro(decoder.decodeString()) } } - - override fun encodeAvroValue(schema: Schema, encoder: ExtendedEncoder, obj: CONVERTED_TYPE) { - return encoder.encodeString(conversion.toAvro(obj)) - } } diff --git a/avro-kotlin-serialization/src/main/kotlin/strategy/GenericRecordSerializationStrategy.kt b/avro-kotlin-serialization/src/main/kotlin/strategy/GenericRecordSerializationStrategy.kt new file mode 100644 index 00000000..24dee04c --- /dev/null +++ b/avro-kotlin-serialization/src/main/kotlin/strategy/GenericRecordSerializationStrategy.kt @@ -0,0 +1,12 @@ +package io.toolisticon.kotlin.avro.serialization.strategy + +import org.apache.avro.generic.GenericRecord +import java.util.function.Predicate +import kotlin.reflect.KClass + +interface GenericRecordSerializationStrategy : Predicate> { + + fun deserialize(serializedType: KClass<*>, data: GenericRecord): T + + fun serialize(data: T): GenericRecord +} diff --git a/avro-kotlin-serialization/src/main/kotlin/strategy/KotlinxDataClassStrategy.kt b/avro-kotlin-serialization/src/main/kotlin/strategy/KotlinxDataClassStrategy.kt new file mode 100644 index 00000000..a2f9a9e4 --- /dev/null +++ b/avro-kotlin-serialization/src/main/kotlin/strategy/KotlinxDataClassStrategy.kt @@ -0,0 +1,22 @@ +package io.toolisticon.kotlin.avro.serialization.strategy + +import io.toolisticon.kotlin.avro.serialization.AvroKotlinSerialization +import io.toolisticon.kotlin.avro.serialization.isKotlinxDataClass +import org.apache.avro.generic.GenericRecord +import kotlin.reflect.KClass + +class KotlinxDataClassStrategy( + private val avroKotlinSerialization: AvroKotlinSerialization +) : GenericRecordSerializationStrategy { + + override fun test(serializedType: KClass<*>): Boolean = serializedType.isKotlinxDataClass() + + @Suppress("UNCHECKED_CAST") + override fun deserialize(serializedType: KClass<*>, data: GenericRecord): T { + return avroKotlinSerialization.genericRecordDecoder(serializedType).decode(data) as T + } + + override fun serialize(data: T): GenericRecord { + return avroKotlinSerialization.genericRecordEncoder().encode(data) + } +} diff --git a/avro-kotlin-serialization/src/main/kotlin/strategy/KotlinxEnumClassStrategy.kt b/avro-kotlin-serialization/src/main/kotlin/strategy/KotlinxEnumClassStrategy.kt new file mode 100644 index 00000000..018f44dd --- /dev/null +++ b/avro-kotlin-serialization/src/main/kotlin/strategy/KotlinxEnumClassStrategy.kt @@ -0,0 +1,22 @@ +package io.toolisticon.kotlin.avro.serialization.strategy + +import io.toolisticon.kotlin.avro.serialization.AvroKotlinSerialization +import io.toolisticon.kotlin.avro.serialization.isKotlinxEnumClass +import org.apache.avro.generic.GenericRecord +import kotlin.reflect.KClass + +class KotlinxEnumClassStrategy( + private val avroKotlinSerialization: AvroKotlinSerialization +) : GenericRecordSerializationStrategy { + + override fun test(serializedType: KClass<*>): Boolean = serializedType.isKotlinxEnumClass() + + @Suppress("UNCHECKED_CAST") + override fun deserialize(serializedType: KClass<*>, data: GenericRecord): T { + return avroKotlinSerialization.genericRecordDecoder(serializedType).decode(data) as T + } + + override fun serialize(data: T): GenericRecord { + return avroKotlinSerialization.genericRecordEncoder().encode(data) + } +} diff --git a/avro-kotlin-serialization/src/main/kotlin/strategy/SpecificRecordBaseStrategy.kt b/avro-kotlin-serialization/src/main/kotlin/strategy/SpecificRecordBaseStrategy.kt new file mode 100644 index 00000000..eff6951b --- /dev/null +++ b/avro-kotlin-serialization/src/main/kotlin/strategy/SpecificRecordBaseStrategy.kt @@ -0,0 +1,20 @@ +package io.toolisticon.kotlin.avro.serialization.strategy + +import io.toolisticon.kotlin.avro.codec.SpecificRecordCodec +import io.toolisticon.kotlin.avro.serialization.isGeneratedSpecificRecordBase +import org.apache.avro.generic.GenericRecord +import org.apache.avro.specific.SpecificRecordBase +import kotlin.reflect.KClass + +class SpecificRecordBaseStrategy : GenericRecordSerializationStrategy { + private val converter = SpecificRecordCodec.specificRecordToGenericRecordConverter() + + override fun test(serializedType: KClass<*>): Boolean = serializedType.isGeneratedSpecificRecordBase() + + override fun deserialize(serializedType: KClass<*>, data: GenericRecord): T { + @Suppress("UNCHECKED_CAST") + return SpecificRecordCodec.genericRecordToSpecificRecordConverter(serializedType.java).convert(data) as T + } + + override fun serialize(data: T): GenericRecord = converter.convert(data as SpecificRecordBase) +} diff --git a/avro-kotlin-serialization/src/test/kotlin/Avro4kSerializationTest.kt b/avro-kotlin-serialization/src/test/kotlin/Avro4kSerializationTest.kt new file mode 100644 index 00000000..aad3eff0 --- /dev/null +++ b/avro-kotlin-serialization/src/test/kotlin/Avro4kSerializationTest.kt @@ -0,0 +1,31 @@ +package io.toolisticon.kotlin.avro.serialization + +import com.github.avrokotlin.avro4k.* +import io.toolisticon.kotlin.avro.AvroKotlin.avroSchemaResolver +import io.toolisticon.kotlin.avro.model.wrapper.AvroSchema +import io.toolisticon.kotlin.avro.serialization._test.DummyEnum +import io.toolisticon.kotlin.avro.serialization.avro4k.Avro4kSchemaRegistry +import io.toolisticon.kotlin.avro.value.SingleObjectEncodedBytes +import kotlinx.serialization.ExperimentalSerializationApi +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.Test + +@OptIn(ExperimentalSerializationApi::class) +internal class Avro4kSerializationTest { + + private val avro4k = Avro.Default + + @Test + fun `serialize enum to single object`() { + val schema = AvroSchema(Avro.schema()) + val resolver = avroSchemaResolver(schema) + + val avro4kSingleObject = AvroSingleObject(schemaRegistry = Avro4kSchemaRegistry(resolver), avro = avro4k) + + val bytes = SingleObjectEncodedBytes.of(avro4kSingleObject.encodeToByteArray(DummyEnum.BAR)) + + val value = avro4kSingleObject.decodeFromByteArray(bytes.value) + + assertThat(value).isEqualTo(DummyEnum.BAR) + } +} diff --git a/avro-kotlin-serialization/src/test/kotlin/AvroKotlinSerializationTest.kt b/avro-kotlin-serialization/src/test/kotlin/AvroKotlinSerializationTest.kt index b17897d6..5c944aa8 100644 --- a/avro-kotlin-serialization/src/test/kotlin/AvroKotlinSerializationTest.kt +++ b/avro-kotlin-serialization/src/test/kotlin/AvroKotlinSerializationTest.kt @@ -1,8 +1,11 @@ package io.toolisticon.kotlin.avro.serialization +import io.toolisticon.kotlin.avro.AvroKotlin.avroSchemaResolver import io.toolisticon.kotlin.avro.model.SchemaType -import io.toolisticon.kotlin.avro.repository.avroSchemaResolver +import io.toolisticon.kotlin.avro.serialization._test.BarString +import io.toolisticon.kotlin.avro.serialization._test.DummyEnum import io.toolisticon.kotlin.avro.serialization._test.Foo +import io.toolisticon.kotlin.avro.serialization._test.barStringSchema import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.Test @@ -10,7 +13,6 @@ internal class AvroKotlinSerializationTest { private val avro = AvroKotlinSerialization() - @Test fun `read schema from Foo`() { val schema = avro.schema(Foo::class) @@ -28,8 +30,6 @@ internal class AvroKotlinSerializationTest { val encoded = avro.encodeSingleObject(foo) - println(encoded.hex.formatted) - val decoded: Foo = avro.decodeFromSingleObject( schemaResolver = avroSchemaResolver(avro.schema(Foo::class)), singleObjectEncodedBytes = encoded @@ -37,4 +37,39 @@ internal class AvroKotlinSerializationTest { assertThat(decoded).isEqualTo(foo) } + + @Test + fun `enum from-to single object encoded`() { + val data = DummyEnum.BAR + + val soe = avro.singleObjectEncoder().encode(data) + + val decoded = avro.singleObjectDecoder().decode(soe) + + assertThat(decoded).isEqualTo(data) + } + + @Test + fun `get schema from BarString`() { + assertThat(avro.cachedSchemaClasses()).isEmpty() + assertThat(avro.cachedSerializerClasses()).isEmpty() + + val schema = avro.schema(BarString::class) + + assertThat(schema.fingerprint).isEqualTo(barStringSchema.fingerprint) + + assertThat(avro.cachedSchemaClasses()).containsExactly(BarString::class) + assertThat(avro.cachedSerializerClasses()).containsExactly(BarString::class) + + assertThat(avro[barStringSchema.fingerprint]).isEqualTo(schema) + + val data = BarString("foo") + val encoded = avro.singleObjectEncoder().encode(data) + + val decoded = avro.singleObjectDecoder().decode(encoded) + + assertThat(decoded).isEqualTo(data) + } + } + diff --git a/avro-kotlin-serialization/src/test/kotlin/_test/BarString.kt b/avro-kotlin-serialization/src/test/kotlin/_test/BarString.kt new file mode 100644 index 00000000..281d1076 --- /dev/null +++ b/avro-kotlin-serialization/src/test/kotlin/_test/BarString.kt @@ -0,0 +1,20 @@ +package io.toolisticon.kotlin.avro.serialization._test + +import io.toolisticon.kotlin.avro.model.wrapper.AvroSchema +import kotlinx.serialization.Serializable +import org.apache.avro.SchemaBuilder + +/** + * It cannot get much simpler than this ... + */ +@Serializable +data class BarString(val name: String) + +val barStringSchema = AvroSchema( + SchemaBuilder.record("BarString") + .namespace("io.toolisticon.kotlin.avro.serialization._test") + .fields() + .requiredString("name") + .endRecord() +) + diff --git a/avro-kotlin-serialization/src/test/kotlin/_test/DummyEnum.kt b/avro-kotlin-serialization/src/test/kotlin/_test/DummyEnum.kt new file mode 100644 index 00000000..86b748b5 --- /dev/null +++ b/avro-kotlin-serialization/src/test/kotlin/_test/DummyEnum.kt @@ -0,0 +1,8 @@ +package io.toolisticon.kotlin.avro.serialization._test + +import kotlinx.serialization.Serializable + +@Serializable +enum class DummyEnum { + FOO, BAR +} diff --git a/avro-kotlin-serialization/src/test/kotlin/serializer/BooleanLogicalTypeSerializerTest.kt b/avro-kotlin-serialization/src/test/kotlin/serializer/BooleanLogicalTypeSerializerTest.kt index c02dff0c..e9ba576f 100644 --- a/avro-kotlin-serialization/src/test/kotlin/serializer/BooleanLogicalTypeSerializerTest.kt +++ b/avro-kotlin-serialization/src/test/kotlin/serializer/BooleanLogicalTypeSerializerTest.kt @@ -1,9 +1,8 @@ package io.toolisticon.kotlin.avro.serialization.serializer +import io.toolisticon.kotlin.avro.AvroKotlin.avroSchemaResolver import io.toolisticon.kotlin.avro.codec.GenericRecordCodec -import io.toolisticon.kotlin.avro.repository.avroSchemaResolver import io.toolisticon.kotlin.avro.serialization.serializer._fixtures.TestBooleanLogicalType -import io.toolisticon.kotlin.avro.serialization.serializer._fixtures.TestDoubleLogicalType import io.toolisticon.kotlin.avro.serialization.serializer._fixtures.avroSerialization import kotlinx.serialization.Serializable import org.assertj.core.api.Assertions.assertThat diff --git a/avro-kotlin-serialization/src/test/kotlin/serializer/BytesLogicalTypeSerializerTest.kt b/avro-kotlin-serialization/src/test/kotlin/serializer/BytesLogicalTypeSerializerTest.kt index cb280a42..64dab568 100644 --- a/avro-kotlin-serialization/src/test/kotlin/serializer/BytesLogicalTypeSerializerTest.kt +++ b/avro-kotlin-serialization/src/test/kotlin/serializer/BytesLogicalTypeSerializerTest.kt @@ -1,7 +1,7 @@ package io.toolisticon.kotlin.avro.serialization.serializer +import io.toolisticon.kotlin.avro.AvroKotlin.avroSchemaResolver import io.toolisticon.kotlin.avro.codec.GenericRecordCodec -import io.toolisticon.kotlin.avro.repository.avroSchemaResolver import io.toolisticon.kotlin.avro.serialization.serializer._fixtures.TestBytesLogicalType import io.toolisticon.kotlin.avro.serialization.serializer._fixtures.avroSerialization import kotlinx.serialization.Serializable diff --git a/avro-kotlin-serialization/src/test/kotlin/serializer/DoubleLogicalTypeSerializerTest.kt b/avro-kotlin-serialization/src/test/kotlin/serializer/DoubleLogicalTypeSerializerTest.kt index 9b276d45..4227c831 100644 --- a/avro-kotlin-serialization/src/test/kotlin/serializer/DoubleLogicalTypeSerializerTest.kt +++ b/avro-kotlin-serialization/src/test/kotlin/serializer/DoubleLogicalTypeSerializerTest.kt @@ -1,7 +1,7 @@ package io.toolisticon.kotlin.avro.serialization.serializer +import io.toolisticon.kotlin.avro.AvroKotlin.avroSchemaResolver import io.toolisticon.kotlin.avro.codec.GenericRecordCodec -import io.toolisticon.kotlin.avro.repository.avroSchemaResolver import io.toolisticon.kotlin.avro.serialization.serializer._fixtures.TestDoubleLogicalType import io.toolisticon.kotlin.avro.serialization.serializer._fixtures.avroSerialization import kotlinx.serialization.Serializable diff --git a/avro-kotlin-serialization/src/test/kotlin/serializer/FloatLogicalTypeSerializerTest.kt b/avro-kotlin-serialization/src/test/kotlin/serializer/FloatLogicalTypeSerializerTest.kt index 065f28a3..449ae849 100644 --- a/avro-kotlin-serialization/src/test/kotlin/serializer/FloatLogicalTypeSerializerTest.kt +++ b/avro-kotlin-serialization/src/test/kotlin/serializer/FloatLogicalTypeSerializerTest.kt @@ -1,7 +1,7 @@ package io.toolisticon.kotlin.avro.serialization.serializer +import io.toolisticon.kotlin.avro.AvroKotlin.avroSchemaResolver import io.toolisticon.kotlin.avro.codec.GenericRecordCodec -import io.toolisticon.kotlin.avro.repository.avroSchemaResolver import io.toolisticon.kotlin.avro.serialization.serializer._fixtures.TestFloatLogicalType import io.toolisticon.kotlin.avro.serialization.serializer._fixtures.avroSerialization import kotlinx.serialization.Serializable diff --git a/avro-kotlin-serialization/src/test/kotlin/serializer/IntLogicalTypeSerializerTest.kt b/avro-kotlin-serialization/src/test/kotlin/serializer/IntLogicalTypeSerializerTest.kt index c288c5f0..49189298 100644 --- a/avro-kotlin-serialization/src/test/kotlin/serializer/IntLogicalTypeSerializerTest.kt +++ b/avro-kotlin-serialization/src/test/kotlin/serializer/IntLogicalTypeSerializerTest.kt @@ -1,7 +1,7 @@ package io.toolisticon.kotlin.avro.serialization.serializer +import io.toolisticon.kotlin.avro.AvroKotlin.avroSchemaResolver import io.toolisticon.kotlin.avro.codec.GenericRecordCodec -import io.toolisticon.kotlin.avro.repository.avroSchemaResolver import io.toolisticon.kotlin.avro.serialization.serializer._fixtures.TestIntLogicalType import io.toolisticon.kotlin.avro.serialization.serializer._fixtures.avroSerialization import kotlinx.serialization.Serializable diff --git a/avro-kotlin-serialization/src/test/kotlin/serializer/LongLogicalTypeSerializerTest.kt b/avro-kotlin-serialization/src/test/kotlin/serializer/LongLogicalTypeSerializerTest.kt index 0ac4dc09..fe2a1bf5 100644 --- a/avro-kotlin-serialization/src/test/kotlin/serializer/LongLogicalTypeSerializerTest.kt +++ b/avro-kotlin-serialization/src/test/kotlin/serializer/LongLogicalTypeSerializerTest.kt @@ -1,7 +1,7 @@ package io.toolisticon.kotlin.avro.serialization.serializer +import io.toolisticon.kotlin.avro.AvroKotlin.avroSchemaResolver import io.toolisticon.kotlin.avro.codec.GenericRecordCodec -import io.toolisticon.kotlin.avro.repository.avroSchemaResolver import io.toolisticon.kotlin.avro.serialization.serializer._fixtures.TestLongLogicalType import io.toolisticon.kotlin.avro.serialization.serializer._fixtures.avroSerialization import kotlinx.serialization.Serializable diff --git a/avro-kotlin-serialization/src/test/kotlin/serializer/StringLogicalTypeSerializerTest.kt b/avro-kotlin-serialization/src/test/kotlin/serializer/StringLogicalTypeSerializerTest.kt index 422c5b60..afe0d068 100644 --- a/avro-kotlin-serialization/src/test/kotlin/serializer/StringLogicalTypeSerializerTest.kt +++ b/avro-kotlin-serialization/src/test/kotlin/serializer/StringLogicalTypeSerializerTest.kt @@ -1,7 +1,7 @@ package serializer +import io.toolisticon.kotlin.avro.AvroKotlin.avroSchemaResolver import io.toolisticon.kotlin.avro.codec.GenericRecordCodec -import io.toolisticon.kotlin.avro.repository.avroSchemaResolver import io.toolisticon.kotlin.avro.serialization.serializer._fixtures.TestStringLogicalType import io.toolisticon.kotlin.avro.serialization.serializer._fixtures.avroSerialization import kotlinx.serialization.Serializable diff --git a/avro-kotlin/pom.xml b/avro-kotlin/pom.xml index 9efc25ed..56f81104 100644 --- a/avro-kotlin/pom.xml +++ b/avro-kotlin/pom.xml @@ -5,7 +5,7 @@ io.toolisticon.kotlin.avro._ avro-kotlin-parent - 1.11.4.4 + 1.11.5.0 ../_mvn/parent/pom.xml diff --git a/avro-kotlin/src/main/kotlin/AvroKotlin.kt b/avro-kotlin/src/main/kotlin/AvroKotlin.kt index 5375466f..491bc943 100644 --- a/avro-kotlin/src/main/kotlin/AvroKotlin.kt +++ b/avro-kotlin/src/main/kotlin/AvroKotlin.kt @@ -1,6 +1,7 @@ package io.toolisticon.kotlin.avro import _ktx.ResourceKtx.resourceUrl +import io.toolisticon.kotlin.avro._ktx.KotlinKtx.head import io.toolisticon.kotlin.avro.declaration.ProtocolDeclaration import io.toolisticon.kotlin.avro.declaration.SchemaDeclaration import io.toolisticon.kotlin.avro.logical.AvroLogicalType @@ -8,8 +9,10 @@ import io.toolisticon.kotlin.avro.model.AvroType import io.toolisticon.kotlin.avro.model.wrapper.AvroProtocol import io.toolisticon.kotlin.avro.model.wrapper.AvroSchema import io.toolisticon.kotlin.avro.repository.AvroSchemaResolver +import io.toolisticon.kotlin.avro.repository.AvroSchemaResolverMap import io.toolisticon.kotlin.avro.value.* import io.toolisticon.kotlin.avro.value.CanonicalName.Companion.toCanonicalName +import org.apache.avro.AvroRuntimeException import org.apache.avro.LogicalTypes import org.apache.avro.Protocol import org.apache.avro.Schema @@ -76,12 +79,42 @@ object AvroKotlin { fun avroType(schema: AvroSchema): AvroType = AvroType.avroType(schema) + fun loadClassForSchema(schema: AvroSchema): KClass { + val nullableJavaClassClass: Class<*>? = specificData.getClass(schema.get()) + + @Suppress("UNCHECKED_CAST") + return if (nullableJavaClassClass != null) + nullableJavaClassClass.kotlin as KClass + else throw AvroRuntimeException("Klass could not be found for ${schema.canonicalName.fqn}") + } + @JvmStatic val genericData: GenericData get() = GenericData() @JvmStatic val specificData: SpecificData get() = SpecificData() + @JvmStatic + fun avroSchemaResolver(schemas: List): AvroSchemaResolverMap { + val (first, other) = schemas.head() + return avroSchemaResolver(first, *(other.toTypedArray())) + } + + @JvmStatic + fun avroSchemaResolver(firstSchema: AvroSchema, vararg otherSchemas: AvroSchema): AvroSchemaResolverMap { + val store = buildMap { + put(firstSchema.fingerprint, firstSchema) + + if (otherSchemas.isNotEmpty()) { + putAll(otherSchemas.associateBy { it.fingerprint }) + } else { + // in case we have a single schema, also provide this for key=NULL, so invoke works + put(AvroFingerprint.NULL, firstSchema) + } + } + + return AvroSchemaResolverMap(store) + } fun canonicalName(namespace: String, name: String): CanonicalName = (namespace to name).toCanonicalName() @@ -165,11 +198,6 @@ object AvroKotlin { ) } - @JvmStatic - fun avroSchemaResolver(schema: Schema) = io.toolisticon.kotlin.avro.repository.avroSchemaResolver( - firstSchema = AvroSchema(schema) - ) - val avroLogicalTypes by lazy { LogicalTypes.getCustomRegisteredTypes().values.filterIsInstance>() .toList() @@ -233,6 +261,4 @@ object AvroKotlin { } fun Result?>.orEmpty(): List = getOrNull() ?: emptyList() - - } diff --git a/avro-kotlin/src/main/kotlin/codec/AvroCodec.kt b/avro-kotlin/src/main/kotlin/codec/AvroCodec.kt index 136adfab..764a32b0 100644 --- a/avro-kotlin/src/main/kotlin/codec/AvroCodec.kt +++ b/avro-kotlin/src/main/kotlin/codec/AvroCodec.kt @@ -5,6 +5,7 @@ import io.toolisticon.kotlin.avro.value.BinaryEncodedBytes import io.toolisticon.kotlin.avro.value.JsonString import io.toolisticon.kotlin.avro.value.SingleObjectEncodedBytes import org.apache.avro.generic.GenericData +import org.apache.avro.generic.GenericRecord import org.apache.avro.io.DecoderFactory import org.apache.avro.io.EncoderFactory import org.apache.avro.specific.SpecificData @@ -46,7 +47,9 @@ object AvroCodec { fun interface SingleObjectEncoder : Encoder fun interface SingleObjectDecoder : Decoder + fun interface GenericRecordEncoder : Encoder + fun interface GenericRecordDecoder : Decoder + interface BinaryEncoder : Encoder interface BinaryDecoder : Decoder - } diff --git a/avro-kotlin/src/main/kotlin/model/wrapper/AvroSchema.kt b/avro-kotlin/src/main/kotlin/model/wrapper/AvroSchema.kt index 8ac3a33f..7325a7d6 100644 --- a/avro-kotlin/src/main/kotlin/model/wrapper/AvroSchema.kt +++ b/avro-kotlin/src/main/kotlin/model/wrapper/AvroSchema.kt @@ -239,23 +239,24 @@ object AvroSchemaChecks { val AvroSchema.isUnion: Boolean get() = get().isUnion val AvroSchema.isUnionType: Boolean get() = SchemaType.UNION == type && isUnion && unionTypes.isNotEmpty() - /** * Check if we can decode using this schema if the encoder used * [writer] schema. * * @param writer - the schema used to encode data - * @return [SchemaCompatibility.SchemaPairCompatibility] with reader=this + * @return [AvroSchemaCompatibility] with reader=this */ - fun AvroSchema.compatibleToReadFrom(writer: AvroSchema): SchemaCompatibility.SchemaPairCompatibility = - SchemaCompatibility.checkReaderWriterCompatibility(get(), writer.get()) + fun AvroSchema.compatibleToReadFrom(writer: AvroSchema): AvroSchemaCompatibility = AvroSchemaCompatibility( + value = SchemaCompatibility.checkReaderWriterCompatibility(get(), writer.get()) + ) /** * Check data encoded using this schema could be decoded from [reader] schema. * * @param reader - the schema to decode the data - * @return [SchemaCompatibility.SchemaPairCompatibility] with writer=this + * @return [AvroSchemaCompatibility] with writer=this */ - fun AvroSchema.compatibleToBeReadFrom(reader: AvroSchema): SchemaCompatibility.SchemaPairCompatibility = - SchemaCompatibility.checkReaderWriterCompatibility(reader.get(), get()) + fun AvroSchema.compatibleToBeReadFrom(reader: AvroSchema) = AvroSchemaCompatibility( + value = SchemaCompatibility.checkReaderWriterCompatibility(reader.get(), get()) + ) } diff --git a/avro-kotlin/src/main/kotlin/repository/AvroSchemaResolver.kt b/avro-kotlin/src/main/kotlin/repository/AvroSchemaResolver.kt index c293f9ee..31e9eb16 100644 --- a/avro-kotlin/src/main/kotlin/repository/AvroSchemaResolver.kt +++ b/avro-kotlin/src/main/kotlin/repository/AvroSchemaResolver.kt @@ -11,7 +11,6 @@ import org.apache.avro.message.SchemaStore */ fun interface AvroSchemaResolver : SchemaStore { - @Throws(MissingSchemaException::class) operator fun get(fingerprint: AvroFingerprint): AvroSchema @@ -21,3 +20,14 @@ fun interface AvroSchemaResolver : SchemaStore { @Throws(MissingSchemaException::class) override fun findByFingerprint(fingerprint: Long): Schema = this[AvroFingerprint(fingerprint)].get() } + + +interface SchemaResolverMap : AvroSchemaResolver, Map { + + @Suppress("PARAMETER_NAME_CHANGED_ON_OVERRIDE") + @kotlin.jvm.Throws(MissingSchemaException::class) + override fun get(fingerprint: AvroFingerprint): AvroSchema +} + +internal fun missingSchemaException(fingerprint: AvroFingerprint) = + MissingSchemaException("Cannot resolve schema for fingerprint: $fingerprint[${fingerprint.value}]") diff --git a/avro-kotlin/src/main/kotlin/repository/AvroSchemaResolverMap.kt b/avro-kotlin/src/main/kotlin/repository/AvroSchemaResolverMap.kt index 5ffea68e..7fffbfdf 100644 --- a/avro-kotlin/src/main/kotlin/repository/AvroSchemaResolverMap.kt +++ b/avro-kotlin/src/main/kotlin/repository/AvroSchemaResolverMap.kt @@ -4,17 +4,19 @@ import io.toolisticon.kotlin.avro.model.wrapper.AvroSchema import io.toolisticon.kotlin.avro.value.AvroFingerprint import org.apache.avro.Schema + data class AvroSchemaResolverMap( private val store: Map = emptyMap() -) : AvroSchemaResolver { +) : SchemaResolverMap, Map by store { companion object { val EMPTY = AvroSchemaResolverMap() } - constructor(schema: Schema) : this((EMPTY + AvroSchema(schema)).store) + constructor(schema: Schema) : this(AvroSchema(schema)) constructor(schema: AvroSchema) : this((EMPTY + schema).store) + @Suppress("PARAMETER_NAME_CHANGED_ON_OVERRIDE") override fun get(fingerprint: AvroFingerprint): AvroSchema = store[fingerprint] ?: throw missingSchemaException( fingerprint ) @@ -26,8 +28,10 @@ data class AvroSchemaResolverMap( } ) - operator fun plus(other: AvroSchemaResolverMap): AvroSchemaResolverMap = copy(store = buildMap { + operator fun plus(other: SchemaResolverMap): AvroSchemaResolverMap = copy(store = buildMap { putAll(store) - putAll(other.store) + putAll(other) }) + + fun toMutableMap(): AvroSchemaResolverMutableMap = AvroSchemaResolverMutableMap.EMPTY + this } diff --git a/avro-kotlin/src/main/kotlin/repository/AvroSchemaResolverMutableMap.kt b/avro-kotlin/src/main/kotlin/repository/AvroSchemaResolverMutableMap.kt new file mode 100644 index 00000000..6a082e7e --- /dev/null +++ b/avro-kotlin/src/main/kotlin/repository/AvroSchemaResolverMutableMap.kt @@ -0,0 +1,39 @@ +package io.toolisticon.kotlin.avro.repository + +import io.toolisticon.kotlin.avro.model.wrapper.AvroSchema +import io.toolisticon.kotlin.avro.value.AvroFingerprint +import org.apache.avro.Schema +import org.apache.avro.message.MissingSchemaException +import java.util.concurrent.ConcurrentHashMap + +/** + * An implementation of [AvroSchemaResolver] that can register new schema definitions + * and keeps all known instances in an in-memory map. + */ +@JvmInline +value class AvroSchemaResolverMutableMap private constructor( + private val store: MutableMap = ConcurrentHashMap() +) : SchemaResolverMap, Map by store { + companion object { + val EMPTY = AvroSchemaResolverMutableMap() + } + + constructor(schema: Schema) : this(AvroSchema(schema)) + constructor(schema: AvroSchema) : this((EMPTY + schema).store) + + @Suppress("PARAMETER_NAME_CHANGED_ON_OVERRIDE") + @Throws(MissingSchemaException::class) + override fun get(fingerprint: AvroFingerprint): AvroSchema = store[fingerprint] ?: throw missingSchemaException( + fingerprint + ) + + operator fun plus(schema: AvroSchema): AvroSchemaResolverMutableMap = apply { + store[schema.fingerprint] = schema + } + + operator fun plus(other: SchemaResolverMap): AvroSchemaResolverMutableMap = apply { + store.putAll(other) + } + + fun toMap() = AvroSchemaResolverMap.EMPTY + this +} diff --git a/avro-kotlin/src/main/kotlin/repository/_resolver.kt b/avro-kotlin/src/main/kotlin/repository/_resolver.kt deleted file mode 100644 index b5584033..00000000 --- a/avro-kotlin/src/main/kotlin/repository/_resolver.kt +++ /dev/null @@ -1,42 +0,0 @@ -@file:JvmName("RepositoryKt") -package io.toolisticon.kotlin.avro.repository - -import io.toolisticon.kotlin.avro._ktx.KotlinKtx.head -import io.toolisticon.kotlin.avro.model.wrapper.AvroSchema -import io.toolisticon.kotlin.avro.value.AvroFingerprint -import org.apache.avro.message.MissingSchemaException - -internal fun missingSchemaException(fingerprint: AvroFingerprint) = - MissingSchemaException("Cannot resolve schema for fingerprint: $fingerprint[${fingerprint.value}]") - -fun avroSchemaResolver(schemas: List): AvroSchemaResolver { - val (first, other) = schemas.head() - return avroSchemaResolver(first, *(other.toTypedArray())) -} - - -fun avroSchemaResolver(firstSchema: AvroSchema, vararg otherSchemas: AvroSchema): AvroSchemaResolver = object : AvroSchemaResolver { - private val store = buildMap { - put(firstSchema.fingerprint, firstSchema) - - if (otherSchemas.isNotEmpty()) { - putAll(otherSchemas.associateBy { it.fingerprint }) - } else { - // in case we have a single schema, also provide this for key=NULL, so invoke works - put(AvroFingerprint.NULL, firstSchema) - } - } - - override fun get(fingerprint: AvroFingerprint): AvroSchema = store[fingerprint] ?: throw missingSchemaException(fingerprint) -} - -operator fun AvroSchemaResolver.plus(other: AvroSchemaResolver): AvroSchemaResolver = AvroSchemaResolver { fingerprint -> - try { - // first try us - this[fingerprint] - } catch (e: MissingSchemaException) { - // then try other - raise exception if still no hit - other[fingerprint] - } -} - diff --git a/avro-kotlin/src/main/kotlin/value/SingleObjectEncodedBytes.kt b/avro-kotlin/src/main/kotlin/value/SingleObjectEncodedBytes.kt index 9fe7b0a7..8c2fefd6 100644 --- a/avro-kotlin/src/main/kotlin/value/SingleObjectEncodedBytes.kt +++ b/avro-kotlin/src/main/kotlin/value/SingleObjectEncodedBytes.kt @@ -44,6 +44,7 @@ value class SingleObjectEncodedBytes private constructor( fun parse(hex: HexString) = of(hex.byteArray) + fun of(bytes: ByteArray) = of(ByteArrayValue(bytes)) } constructor(fingerprint: AvroFingerprint, payload: BinaryEncodedBytes) : this(fingerprint to payload) diff --git a/avro-kotlin/src/main/kotlin/value/_compatibility.kt b/avro-kotlin/src/main/kotlin/value/_compatibility.kt new file mode 100644 index 00000000..557d8685 --- /dev/null +++ b/avro-kotlin/src/main/kotlin/value/_compatibility.kt @@ -0,0 +1,54 @@ +package io.toolisticon.kotlin.avro.value + +import io.toolisticon.kotlin.avro.model.wrapper.AvroSchema +import io.toolisticon.kotlin.avro.model.wrapper.AvroSchemaChecks.compatibleToReadFrom +import org.apache.avro.SchemaCompatibility.SchemaCompatibilityResult +import org.apache.avro.SchemaCompatibility.SchemaPairCompatibility +import java.util.concurrent.ConcurrentHashMap + +/** + * Used as a key in [AvroSchemaCompatibilityMap] to cache [AvroSchemaCompatibility] results. + */ +@JvmInline +value class AvroFingerprintPair private constructor(override val value: Pair) : PairType { + companion object { + fun of(writerSchema: AvroSchema, readerSchema: AvroSchema) = AvroFingerprintPair(writerSchema.fingerprint, readerSchema.fingerprint) + } + + constructor(writer: AvroFingerprint, reader: AvroFingerprint) : this(writer to reader) + + val writerSchema: AvroFingerprint get() = value.first + val readerSchema: AvroFingerprint get() = value.second +} + +/** + * Wraps apache-avro [SchemaPairCompatibility] and allows simple(typesafe access to + * helper functions and derived attributes. + */ +@JvmInline +value class AvroSchemaCompatibility(override val value: SchemaPairCompatibility) : ValueType { + val result: SchemaCompatibilityResult get() = value.result + + val isCompatible: Boolean get() = result.incompatibilities.isEmpty() +} + +/** + * Mutable cache for [AvroSchemaCompatibility], so based on a writer- and reader-schema + * fingerprint, we only calculate once. + */ +@JvmInline +value class AvroSchemaCompatibilityMap( + private val value: MutableMap = ConcurrentHashMap() +) { + + fun compatibleToReadFrom(writerSchema: AvroSchema, readerSchema: AvroSchema): AvroSchemaCompatibility { + val key = AvroFingerprintPair.of(writerSchema, readerSchema) + + return value.computeIfAbsent(key) { _ -> readerSchema.compatibleToReadFrom(writerSchema) } + } + + fun isCompatible(writerSchema: AvroFingerprint, readerSchema: AvroFingerprint): Boolean = + value.getOrDefault(AvroFingerprintPair(writerSchema, readerSchema), null) + ?.isCompatible + ?: false +} diff --git a/avro-kotlin/src/test/kotlin/AvroSchemaResolverTest.kt b/avro-kotlin/src/test/kotlin/AvroSchemaResolverTest.kt index a21f23fd..9ced74cd 100644 --- a/avro-kotlin/src/test/kotlin/AvroSchemaResolverTest.kt +++ b/avro-kotlin/src/test/kotlin/AvroSchemaResolverTest.kt @@ -1,7 +1,7 @@ package io.toolisticon.kotlin.avro +import io.toolisticon.kotlin.avro.AvroKotlin.avroSchemaResolver import io.toolisticon.kotlin.avro.TestFixtures.BankAccountCreatedFixtures.SCHEMA_BANK_ACCOUNT_CREATED -import io.toolisticon.kotlin.avro.repository.avroSchemaResolver import io.toolisticon.kotlin.avro.value.AvroFingerprint import org.apache.avro.message.MissingSchemaException import org.assertj.core.api.Assertions.assertThat diff --git a/avro-kotlin/src/test/kotlin/TestFixtures.kt b/avro-kotlin/src/test/kotlin/TestFixtures.kt index 4545d981..e683fc7b 100644 --- a/avro-kotlin/src/test/kotlin/TestFixtures.kt +++ b/avro-kotlin/src/test/kotlin/TestFixtures.kt @@ -5,6 +5,7 @@ import _ktx.ResourceKtx.loadJsonString import _ktx.ResourceKtx.resourceUrl import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper import io.toolisticon.kotlin.avro.model.SchemaType +import io.toolisticon.kotlin.avro.model.wrapper.AvroProtocol import io.toolisticon.kotlin.avro.model.wrapper.AvroSchema import io.toolisticon.kotlin.avro.value.* import lib.test.event.BankAccountCreated @@ -28,9 +29,11 @@ object TestFixtures { fun parseSchema(json: JsonString): Schema = Schema.Parser().parse(json.inputStream()) fun loadSchema(resource: String): Schema = parseSchema(loadJsonString(resource)) + fun loadAvroSchema(resource: String): AvroSchema = AvroSchema(loadSchema(resource)) fun parseProtocol(json: JsonString): Protocol = Protocol.parse(json.inputStream()) fun loadProtocol(resource: String): Protocol = parseProtocol(loadJsonString(resource)) + fun loadAvroProtocol(resource: String): AvroProtocol = AvroProtocol(loadProtocol(resource)) /** * this schema contains 5 types: @@ -131,4 +134,34 @@ object TestFixtures { resourceUrl("org.apache.avro/schema/json.avsc").openStream() ) } + + object DummyEvents { + val jsonSchema01 = JsonString.of( + """ + { + "type": "record", + "namespace": "upcaster.itest", + "name": "DummyEvent", + "revision": "1", + "fields": [ + { + "name": "value01", + "type": { + "type": "string", + "avro.java.string": "String" + } + } + ] + } + """.trimIndent() + ) + + val SCHEMA_EVENT_01: AvroSchema = AvroSchema.of(jsonSchema01) + + val SCHEMA_EVENT_10 = AvroSchema.of( + JsonString.of("{\"type\":\"record\",\"name\":\"DummyEvent\",\"namespace\":\"upcaster.itest\",\"fields\":[{\"name\":\"value01\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"}},{\"name\":\"value10\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"}}],\"revision\":\"10\"}") + ) + + val registry = AvroKotlin.avroSchemaResolver(listOf(SCHEMA_EVENT_01, SCHEMA_EVENT_10)) + } } diff --git a/avro-kotlin/src/test/kotlin/_test/FooStringTest.kt b/avro-kotlin/src/test/kotlin/_test/FooStringTest.kt index 9498e58b..577f7744 100644 --- a/avro-kotlin/src/test/kotlin/_test/FooStringTest.kt +++ b/avro-kotlin/src/test/kotlin/_test/FooStringTest.kt @@ -1,10 +1,10 @@ package io.toolisticon.kotlin.avro._test import io.toolisticon.kotlin.avro.AvroKotlin +import io.toolisticon.kotlin.avro.AvroKotlin.avroSchemaResolver import io.toolisticon.kotlin.avro.model.wrapper.AvroSchema import io.toolisticon.kotlin.avro.model.wrapper.AvroSchemaChecks.compatibleToBeReadFrom import io.toolisticon.kotlin.avro.model.wrapper.AvroSchemaChecks.compatibleToReadFrom -import io.toolisticon.kotlin.avro.repository.avroSchemaResolver import org.apache.avro.SchemaCompatibility import org.apache.avro.generic.GenericData import org.assertj.core.api.Assertions.assertThat diff --git a/avro-kotlin/src/test/kotlin/codec/GenericRecordCodecTest.kt b/avro-kotlin/src/test/kotlin/codec/GenericRecordCodecTest.kt index b7d3a64a..404c7351 100644 --- a/avro-kotlin/src/test/kotlin/codec/GenericRecordCodecTest.kt +++ b/avro-kotlin/src/test/kotlin/codec/GenericRecordCodecTest.kt @@ -1,9 +1,9 @@ package io.toolisticon.kotlin.avro.codec +import io.toolisticon.kotlin.avro.AvroKotlin.avroSchemaResolver import io.toolisticon.kotlin.avro._test.FooString import io.toolisticon.kotlin.avro._test.FooString2 import io.toolisticon.kotlin.avro.codec.GenericRecordCodec.convert -import io.toolisticon.kotlin.avro.repository.avroSchemaResolver import io.toolisticon.kotlin.avro.value.SingleObjectEncodedBytes import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.Test diff --git a/avro-kotlin/src/test/kotlin/codec/SpecificRecordCodecTest.kt b/avro-kotlin/src/test/kotlin/codec/SpecificRecordCodecTest.kt index e56db9dc..06ae0f9f 100644 --- a/avro-kotlin/src/test/kotlin/codec/SpecificRecordCodecTest.kt +++ b/avro-kotlin/src/test/kotlin/codec/SpecificRecordCodecTest.kt @@ -1,8 +1,8 @@ package io.toolisticon.kotlin.avro.codec +import io.toolisticon.kotlin.avro.AvroKotlin.avroSchemaResolver import io.toolisticon.kotlin.avro.TestFixtures.BankAccountCreatedFixtures import io.toolisticon.kotlin.avro._test.BankAccountCreatedData -import io.toolisticon.kotlin.avro.repository.avroSchemaResolver import lib.test.event.BankAccountCreated import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.Test diff --git a/avro-kotlin/src/test/kotlin/repository/AvroSchemaResolverMapTest.kt b/avro-kotlin/src/test/kotlin/repository/AvroSchemaResolverMapTest.kt deleted file mode 100644 index e83e701e..00000000 --- a/avro-kotlin/src/test/kotlin/repository/AvroSchemaResolverMapTest.kt +++ /dev/null @@ -1,5 +0,0 @@ -package io.toolisticon.kotlin.avro.repository - -import org.junit.jupiter.api.Assertions.* - -class AvroSchemaResolverMapTest diff --git a/avro-kotlin/src/test/kotlin/repository/AvroSchemaResolverMutableMapTest.kt b/avro-kotlin/src/test/kotlin/repository/AvroSchemaResolverMutableMapTest.kt new file mode 100644 index 00000000..0c1995bd --- /dev/null +++ b/avro-kotlin/src/test/kotlin/repository/AvroSchemaResolverMutableMapTest.kt @@ -0,0 +1,29 @@ +package io.toolisticon.kotlin.avro.repository + +import io.toolisticon.kotlin.avro.TestFixtures.loadAvroSchema +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.Test + + +internal class AvroSchemaResolverMutableMapTest { + + @Test + fun `empty instance`() { + assertThat(AvroSchemaResolverMutableMap.EMPTY).isEmpty() + } + + @Test + fun `init with single schema`() { + val schema = loadAvroSchema("avro/lib/test/event/BankAccountCreated.avsc") + assertThat(AvroSchemaResolverMutableMap(schema)).hasSize(1) + } + + @Test + fun `add schema`() { + val schema = loadAvroSchema("avro/lib/test/event/BankAccountCreated.avsc") + val resolver = AvroSchemaResolverMutableMap(schema) + assertThat(resolver).hasSize(1) + resolver.plus(loadAvroSchema("avro/lib/test/dummy/NestedDummy.avsc")) + assertThat(resolver).hasSize(2) + } +} diff --git a/avro-kotlin/src/test/kotlin/value/AvroSchemaCompatibilityMapTest.kt b/avro-kotlin/src/test/kotlin/value/AvroSchemaCompatibilityMapTest.kt new file mode 100644 index 00000000..1dcbcacf --- /dev/null +++ b/avro-kotlin/src/test/kotlin/value/AvroSchemaCompatibilityMapTest.kt @@ -0,0 +1,20 @@ +package io.toolisticon.kotlin.avro.value + +import io.toolisticon.kotlin.avro.TestFixtures.DummyEvents +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.Test + +internal class AvroSchemaCompatibilityMapTest { + + @Test + fun `schemas are incompatible`() { + val cache = AvroSchemaCompatibilityMap() + + val result = cache.compatibleToReadFrom(DummyEvents.SCHEMA_EVENT_01, DummyEvents.SCHEMA_EVENT_10) + + assertThat(result.isCompatible).isFalse() + + // cached result + assertThat(cache.isCompatible(DummyEvents.SCHEMA_EVENT_01.fingerprint, DummyEvents.SCHEMA_EVENT_10.fingerprint)).isFalse() + } +} diff --git a/lib/avro-compiler/pom.xml b/lib/avro-compiler/pom.xml index 6f48dee4..03349ff4 100644 --- a/lib/avro-compiler/pom.xml +++ b/lib/avro-compiler/pom.xml @@ -5,7 +5,7 @@ io.toolisticon.kotlin.avro._ avro-lib-root - 1.11.4.4 + 1.11.5.0 io.toolisticon.kotlin.avro.lib diff --git a/lib/avro-ipc/pom.xml b/lib/avro-ipc/pom.xml index cca97aa4..27693c79 100644 --- a/lib/avro-ipc/pom.xml +++ b/lib/avro-ipc/pom.xml @@ -5,7 +5,7 @@ io.toolisticon.kotlin.avro._ avro-lib-root - 1.11.4.4 + 1.11.5.0 io.toolisticon.kotlin.avro.lib diff --git a/lib/avro/pom.xml b/lib/avro/pom.xml index f658b92f..7fcf69a5 100644 --- a/lib/avro/pom.xml +++ b/lib/avro/pom.xml @@ -5,7 +5,7 @@ io.toolisticon.kotlin.avro._ avro-lib-root - 1.11.4.4 + 1.11.5.0 io.toolisticon.kotlin.avro.lib @@ -58,12 +58,24 @@ org.apache.commons commons-compress + + + org.apache.commons + commons-lang3 + + org.slf4j slf4j-api + + + org.apache.commons + commons-lang3 + ${commons-lang3.version} + diff --git a/lib/avro4k-core/pom.xml b/lib/avro4k-core/pom.xml index 3d5298d9..c3291770 100644 --- a/lib/avro4k-core/pom.xml +++ b/lib/avro4k-core/pom.xml @@ -5,7 +5,7 @@ io.toolisticon.kotlin.avro._ avro-lib-root - 1.11.4.4 + 1.11.5.0 io.toolisticon.kotlin.avro.lib @@ -30,8 +30,12 @@ avro - org.xerial.snappy - snappy-java + org.jetbrains.kotlinx + kotlinx-serialization-json-jvm + + + org.jetbrains.kotlinx + kotlinx-serialization-core-jvm @@ -40,8 +44,20 @@ io.toolisticon.kotlin.avro.lib avro ${project.version} + compile + + org.jetbrains.kotlinx + kotlinx-serialization-core-jvm + compile + + + + org.jetbrains.kotlinx + kotlinx-serialization-json-jvm + runtime + diff --git a/lib/pom.xml b/lib/pom.xml index 8fb893af..213401e5 100644 --- a/lib/pom.xml +++ b/lib/pom.xml @@ -5,7 +5,7 @@ io.toolisticon.kotlin.avro._ avro-kotlin-root - 1.11.4.4 + 1.11.5.0 avro-lib-root @@ -15,9 +15,9 @@ 1.11.3 - 1.10.1 + 2.0.0 1.26.2 - 3.14.0 + 3.15.0 1.12.0 2.0.13 diff --git a/pom.xml b/pom.xml index 039471bf..7f475613 100644 --- a/pom.xml +++ b/pom.xml @@ -5,13 +5,13 @@ io.toolisticon.maven.parent maven-parent-kotlin-base - 2024.6.1 + 2024.7.1 io.toolisticon.kotlin.avro._ avro-kotlin-root - 1.11.4.4 + 1.11.5.0 pom: ${project.artifactId} Root of opinionated extension functions and helpers for using Apache Avro with Kotlin.