From 354b367baf343c774f20d6fc526816ec74753e09 Mon Sep 17 00:00:00 2001 From: "Ilya.Usov" Date: Wed, 10 Jul 2024 16:49:59 +0200 Subject: [PATCH] Support saving the list of marshallers during generation to a file Support passing MarshallersProvider to avoid greedy registration of all serializers on the startup, and just search for them when they are needed. It also solves the `Maybe you forgot to invoke 'register()' method` problem --- .../rd/framework/MarshallersProvider.kt | 44 +++++++++++++++ .../com/jetbrains/rd/framework/Serializers.kt | 27 ++++++--- .../generator/gradle/GradleGenerationSpec.kt | 3 +- .../rd/generator/nova/GenerationSpec.kt | 11 ++-- .../jetbrains/rd/generator/nova/Generators.kt | 23 +++++++- .../rd/generator/nova/MarshallersCollector.kt | 39 +++++++++++++ .../rd/generator/nova/cpp/Cpp17Generator.kt | 2 +- .../nova/csharp/CSharp50Generator.kt | 2 +- .../nova/kotlin/Kotlin11Generator.kt | 55 +++++++++++-------- 9 files changed, 165 insertions(+), 41 deletions(-) create mode 100644 rd-kt/rd-framework/src/main/kotlin/com/jetbrains/rd/framework/MarshallersProvider.kt create mode 100644 rd-kt/rd-gen/src/main/kotlin/com/jetbrains/rd/generator/nova/MarshallersCollector.kt diff --git a/rd-kt/rd-framework/src/main/kotlin/com/jetbrains/rd/framework/MarshallersProvider.kt b/rd-kt/rd-framework/src/main/kotlin/com/jetbrains/rd/framework/MarshallersProvider.kt new file mode 100644 index 000000000..b022d07c9 --- /dev/null +++ b/rd-kt/rd-framework/src/main/kotlin/com/jetbrains/rd/framework/MarshallersProvider.kt @@ -0,0 +1,44 @@ +package com.jetbrains.rd.framework + +import com.jetbrains.rd.util.error +import com.jetbrains.rd.util.getLogger +import java.io.InputStream + +interface MarshallersProvider { + object Dummy : MarshallersProvider { + override fun getMarshaller(id: RdId): IMarshaller<*>? = null + } + + companion object { + fun extractMarshallers( + stream: InputStream, + classsLoader: ClassLoader + ): List> = stream.reader().useLines { lines: Sequence -> + lines.map> { it.split(":") }.map, LazyCompanionMarshaller> { + LazyCompanionMarshaller(RdId(it.first().toLong()), classsLoader, it.last()) + }.toList() + } + } + + fun getMarshaller(id: RdId): IMarshaller<*>? +} + +class AggregatedMarshallersProvider(val providers: Sequence) : MarshallersProvider { + override fun getMarshaller(id: RdId): IMarshaller<*>? { + return providers.mapNotNull { it.getMarshaller(id) }.firstOrNull() + } +} + +abstract class MarhallersProviderFromResourcesBase(val resourceName: String) : MarshallersProvider { + private val map: Map> by lazy { + val classLoader = javaClass.classLoader + val resource = classLoader.getResourceAsStream(resourceName) ?: run { + getLogger(this::class).error { "$resourceName is not found" } + return@lazy emptyMap() + } + + MarshallersProvider.extractMarshallers(resource, classsLoader = classLoader).associateBy { it.id } + } + + override fun getMarshaller(id: RdId): IMarshaller<*>? = map[id] +} \ No newline at end of file diff --git a/rd-kt/rd-framework/src/main/kotlin/com/jetbrains/rd/framework/Serializers.kt b/rd-kt/rd-framework/src/main/kotlin/com/jetbrains/rd/framework/Serializers.kt index 8d7350301..bf760c8e1 100644 --- a/rd-kt/rd-framework/src/main/kotlin/com/jetbrains/rd/framework/Serializers.kt +++ b/rd-kt/rd-framework/src/main/kotlin/com/jetbrains/rd/framework/Serializers.kt @@ -5,9 +5,7 @@ import com.jetbrains.rd.framework.impl.RdSecureString import com.jetbrains.rd.util.* import com.jetbrains.rd.util.hash.getPlatformIndependentHash import com.jetbrains.rd.util.lifetime.Lifetime -import java.util.concurrent.TimeUnit import kotlin.reflect.KClass -import kotlin.reflect.jvm.jvmName import kotlin.time.Duration import kotlin.time.DurationUnit import kotlin.time.toDuration @@ -16,12 +14,14 @@ private const val notRegisteredErrorMessage = "Maybe you forgot to invoke 'regis "Usually it should be done automatically during 'bind()' invocation but in complex cases you should do it manually." -@Suppress("UNCHECKED_CAST") -class Serializers : ISerializers { - +class Serializers(private val provider: MarshallersProvider) : ISerializers { companion object { private val backgroundRegistrar = createBackgroundScheduler(Lifetime.Eternal, "SerializersBackgroundRegistrar") } + + @Deprecated("Use an overload with MarshallersProvider") + constructor() : this(MarshallersProvider.Dummy) + override fun registerSerializersOwnerOnce(serializersOwner: ISerializersOwner) { backgroundRegistrar.invokeOrQueue { val key = serializersOwner::class @@ -54,7 +54,7 @@ class Serializers : ISerializers { val id = serializer.id val existing = marshallers[id] if (existing != null) { - require(existing.fqn == serializer.fqn) { "Can't register ${serializer.fqn} with id: $id, already registered: ${serializer.fqn}" } + assertSerializersAreTheSame(existing, serializer, id) } else { Protocol.initializationLogger.trace { "Registering type ${serializer.fqn}, id = $id" } marshallers[id] = serializer @@ -63,6 +63,10 @@ class Serializers : ISerializers { } } + private fun assertSerializersAreTheSame(existing: IMarshaller<*>, new: IMarshaller<*>, id: RdId) { + require(existing.fqn == new.fqn) { "Can't register ${new.fqn} with id: $id, already registered: ${new.fqn}" } + } + override fun get(id: RdId): IMarshaller<*>? { return marshallers[id] } @@ -75,7 +79,14 @@ class Serializers : ISerializers { val size = stream.readInt() stream.checkAvailable(size) - val reader = marshallers[id] + val reader = marshallers[id] ?: run { + provider.getMarshaller(id)?.let { newMarshaller -> + marshallers.putIfAbsent(id, newMarshaller)?.also { existing -> + assertSerializersAreTheSame(existing, newMarshaller, id) + } ?: newMarshaller + } + } + if (reader == null) { if (abstractDeclaration == null) { throw IllegalStateException("Can't find reader by id: $id. $notRegisteredErrorMessage") @@ -103,7 +114,7 @@ class Serializers : ISerializers { private fun getWriter(clazz: KClass): IMarshaller { val marshaller = writers.getOrPut(clazz) { val id = RdId(clazz.simpleName.getPlatformIndependentHash()) - marshallers[id] ?: cantFindWriter(clazz) + marshallers[id] ?: provider.getMarshaller(id) ?: cantFindWriter(clazz) } return marshaller as? IMarshaller ?: cantFindWriter(clazz) diff --git a/rd-kt/rd-gen/src/gradlePlugin/kotlin/com/jetbrains/rd/generator/gradle/GradleGenerationSpec.kt b/rd-kt/rd-gen/src/gradlePlugin/kotlin/com/jetbrains/rd/generator/gradle/GradleGenerationSpec.kt index 9ff856ee9..62f9d1650 100644 --- a/rd-kt/rd-gen/src/gradlePlugin/kotlin/com/jetbrains/rd/generator/gradle/GradleGenerationSpec.kt +++ b/rd-kt/rd-gen/src/gradlePlugin/kotlin/com/jetbrains/rd/generator/gradle/GradleGenerationSpec.kt @@ -7,8 +7,9 @@ class GradleGenerationSpec { var namespace = "" var directory = "" var generatedFileSuffix = ".Generated" + var marshallersFile: String? = "" override fun toString(): String { - return "$language||$transform||$root||$namespace||$directory||$generatedFileSuffix" + return "$language||$transform||$root||$namespace||$directory||$generatedFileSuffix||${marshallersFile}" } } \ No newline at end of file diff --git a/rd-kt/rd-gen/src/main/kotlin/com/jetbrains/rd/generator/nova/GenerationSpec.kt b/rd-kt/rd-gen/src/main/kotlin/com/jetbrains/rd/generator/nova/GenerationSpec.kt index d4ffda63c..6af3c2dae 100644 --- a/rd-kt/rd-gen/src/main/kotlin/com/jetbrains/rd/generator/nova/GenerationSpec.kt +++ b/rd-kt/rd-gen/src/main/kotlin/com/jetbrains/rd/generator/nova/GenerationSpec.kt @@ -11,7 +11,8 @@ data class GenerationSpec( var root: String = "", var namespace: String = "", var directory: String = "", - var generatedFileSuffix: String = ".Generated" + var generatedFileSuffix: String = ".Generated", + var marshallersFile: String? = null, ) { companion object { fun loadFrom(file: File): List { @@ -19,10 +20,12 @@ data class GenerationSpec( val lines = file.readLines(Charsets.UTF_8) for (line in lines) { val parts = line.split("||") - result.add(GenerationSpec(parts[0], parts[1], parts[2], parts[3], parts[4], parts[5])) + result.add(GenerationSpec(parts[0], parts[1], parts[2], parts[3], parts[4], parts[5], parts[6].nullIfEmpty())) } return result } + + fun String.nullIfEmpty(): String? = if (this.isBlank()) null else this } fun toGeneratorAndRoot(availableRoots: List): IGeneratorAndRoot { @@ -34,8 +37,8 @@ data class GenerationSpec( else -> throw GeneratorException("Unknown flow transform type ${transform}, use 'asis', 'reversed' or 'symmetric'") } val generator = when (language) { - "kotlin" -> Kotlin11Generator(flowTransform, namespace, File(directory), generatedFileSuffix) - "csharp" -> CSharp50Generator(flowTransform, namespace, File(directory), generatedFileSuffix) + "kotlin" -> Kotlin11Generator(flowTransform, namespace, File(directory), generatedFileSuffix, marhsallersFile = marshallersFile?.let { File(it) }) + "csharp" -> CSharp50Generator(flowTransform, namespace, File(directory), generatedFileSuffix) // todo support for C# "cpp" -> Cpp17Generator(flowTransform, namespace, File(directory), generatedFileSuffix) else -> throw GeneratorException("Unknown language $language, use 'kotlin' or 'csharp' or 'cpp'") } diff --git a/rd-kt/rd-gen/src/main/kotlin/com/jetbrains/rd/generator/nova/Generators.kt b/rd-kt/rd-gen/src/main/kotlin/com/jetbrains/rd/generator/nova/Generators.kt index 44e269a1f..871c26eab 100644 --- a/rd-kt/rd-gen/src/main/kotlin/com/jetbrains/rd/generator/nova/Generators.kt +++ b/rd-kt/rd-gen/src/main/kotlin/com/jetbrains/rd/generator/nova/Generators.kt @@ -44,7 +44,7 @@ fun fail(msg: String) : Nothing { throw GeneratorException(msg) } /** * Base class for generators to deduplicate common logic */ -abstract class GeneratorBase(protected open val flowTransform: FlowTransform, protected val generatedFileSuffix: String) : IGenerator { +abstract class GeneratorBase(protected open val flowTransform: FlowTransform, protected val generatedFileSuffix: String, val marhsallersFile: File? = null,) : IGenerator { object AllowDeconstruct: ISetting /** @@ -53,16 +53,33 @@ abstract class GeneratorBase(protected open val flowTransform: FlowTransform, pr object AcceptsGenerator: ISetting<(IGenerator) -> Boolean, Toplevel> - protected abstract fun realGenerate(toplevels: List) + protected abstract fun realGenerate(toplevels: List, collector: MarshallersCollector) override fun generate(toplevels: List) { val preparedToplevels = toplevels .filter { it.getSetting(AcceptsGenerator)?.invoke(this) ?: true } .sortedBy { it.name } - realGenerate(preparedToplevels) + withContext { + realGenerate(preparedToplevels, it) + } } + private fun withContext(action: (MarshallersCollector) -> Unit) { + val file = marhsallersFile + if (file != null) { + val context = RealMarshallersCollector(file) + try { + action(context) + }finally { + context.close() + } + } else { + action(DisabledMarshallersCollector) + } + } + + protected open fun unknowns(declaredTypes: Iterable): Collection { return declaredTypes.mapNotNull { val unknown: Declaration? = unknown(it) diff --git a/rd-kt/rd-gen/src/main/kotlin/com/jetbrains/rd/generator/nova/MarshallersCollector.kt b/rd-kt/rd-gen/src/main/kotlin/com/jetbrains/rd/generator/nova/MarshallersCollector.kt new file mode 100644 index 000000000..f211371c1 --- /dev/null +++ b/rd-kt/rd-gen/src/main/kotlin/com/jetbrains/rd/generator/nova/MarshallersCollector.kt @@ -0,0 +1,39 @@ +package com.jetbrains.rd.generator.nova + +import com.jetbrains.rd.util.hash.getPlatformIndependentHash +import java.io.File + + +interface MarshallersCollector { + val shouldGenerateRegistrations: Boolean + + fun addMarshaller(namespace: String, name: String) +} + +object DisabledMarshallersCollector : MarshallersCollector { + override val shouldGenerateRegistrations: Boolean + get() = true + + override fun addMarshaller(namespace: String, name: String) { + } +} + +class RealMarshallersCollector(val marshallersFile: File) : MarshallersCollector { + private val marshallers = mutableSetOf() + + override val shouldGenerateRegistrations: Boolean + get() = false // We may want to add a separate setting here, but for now just disable it + + override fun addMarshaller(namespace: String, name: String) { + marshallers.add("${name.getPlatformIndependentHash()}:${namespace}.${name}") + } + + fun close() { + marshallersFile.parentFile.mkdirs() + marshallersFile.writer().use { writer -> + marshallers.sorted().forEach { + writer.append(it).append("\n") + } + } + } +} \ No newline at end of file diff --git a/rd-kt/rd-gen/src/main/kotlin/com/jetbrains/rd/generator/nova/cpp/Cpp17Generator.kt b/rd-kt/rd-gen/src/main/kotlin/com/jetbrains/rd/generator/nova/cpp/Cpp17Generator.kt index e4371a625..f430213e9 100644 --- a/rd-kt/rd-gen/src/main/kotlin/com/jetbrains/rd/generator/nova/cpp/Cpp17Generator.kt +++ b/rd-kt/rd-gen/src/main/kotlin/com/jetbrains/rd/generator/nova/cpp/Cpp17Generator.kt @@ -902,7 +902,7 @@ open class Cpp17Generator( +"//------------------------------------------------------------------------------" } - override fun realGenerate(toplevels: List) { + override fun realGenerate(toplevels: List, collector: MarshallersCollector) { if (toplevels.isEmpty()) return val root = toplevels.first().root diff --git a/rd-kt/rd-gen/src/main/kotlin/com/jetbrains/rd/generator/nova/csharp/CSharp50Generator.kt b/rd-kt/rd-gen/src/main/kotlin/com/jetbrains/rd/generator/nova/csharp/CSharp50Generator.kt index 870e1167a..9b3758b09 100644 --- a/rd-kt/rd-gen/src/main/kotlin/com/jetbrains/rd/generator/nova/csharp/CSharp50Generator.kt +++ b/rd-kt/rd-gen/src/main/kotlin/com/jetbrains/rd/generator/nova/csharp/CSharp50Generator.kt @@ -360,7 +360,7 @@ open class CSharp50Generator( //generation - override fun realGenerate(toplevels: List) { + override fun realGenerate(toplevels: List, collector: MarshallersCollector) { toplevels.forEach { tl -> tl.fsPath.bufferedWriter().use { writer -> PrettyPrinter().apply { diff --git a/rd-kt/rd-gen/src/main/kotlin/com/jetbrains/rd/generator/nova/kotlin/Kotlin11Generator.kt b/rd-kt/rd-gen/src/main/kotlin/com/jetbrains/rd/generator/nova/kotlin/Kotlin11Generator.kt index 73549cfd2..ce5769189 100644 --- a/rd-kt/rd-gen/src/main/kotlin/com/jetbrains/rd/generator/nova/kotlin/Kotlin11Generator.kt +++ b/rd-kt/rd-gen/src/main/kotlin/com/jetbrains/rd/generator/nova/kotlin/Kotlin11Generator.kt @@ -26,8 +26,9 @@ open class Kotlin11Generator( flowTransform: FlowTransform, private val defaultNamespace: String, override val folder: File, - generatedFileSuffix: String = ".Generated" -) : GeneratorBase(flowTransform, generatedFileSuffix) { + generatedFileSuffix: String = ".Generated", + marhsallersFile: File? = null, +) : GeneratorBase(flowTransform, generatedFileSuffix, marhsallersFile) { //language specific properties object Namespace : ISetting @@ -292,11 +293,8 @@ open class Kotlin11Generator( else -> "__${name}Serializer" } - - - //generation - override fun realGenerate(toplevels: List) { + override fun realGenerate(toplevels: List, collector: MarshallersCollector) { toplevels.forEach { tl -> tl.fsPath.bufferedWriter().use { writer -> PrettyPrinter().apply { @@ -304,7 +302,7 @@ open class Kotlin11Generator( step = 4 //actual generation - file(tl) + file(tl, collector) writer.write(toString()) } @@ -316,7 +314,7 @@ open class Kotlin11Generator( - protected open fun PrettyPrinter.file(tl : Toplevel) { + protected open fun PrettyPrinter.file(tl: Toplevel, collector: MarshallersCollector) { namespace(tl) println() @@ -327,12 +325,12 @@ open class Kotlin11Generator( val types = tl.declaredTypes + unknowns(tl.declaredTypes) if (tl.isLibrary) - libdef(tl, types) + libdef(tl, types, collector) else - typedef(tl) + typedef(tl, collector) types.sortedBy { it.name }.forEach { type -> - typedef(type) + typedef(type, collector) } } @@ -369,14 +367,14 @@ open class Kotlin11Generator( } - protected open fun PrettyPrinter.libdef(decl: Toplevel, types: List) { + protected open fun PrettyPrinter.libdef(decl: Toplevel, types: List, collector: MarshallersCollector) { if (decl.getSetting(Intrinsic) != null) return block("object ${decl.name} : ISerializersOwner ") { - registerSerializersTrait(decl, types) + registerSerializersTrait(decl, types, collector) } } - protected open fun PrettyPrinter.typedef(decl: Declaration) { + protected open fun PrettyPrinter.typedef(decl: Declaration, collector: MarshallersCollector) { if (decl.getSetting(Intrinsic) != null || decl is Context) return println() @@ -415,7 +413,7 @@ open class Kotlin11Generator( block("") { + "//companion" - companionTrait(decl) + companionTrait(decl, collector) + "//fields" fieldsTrait(decl) + "//methods" @@ -515,9 +513,10 @@ open class Kotlin11Generator( } } - protected fun PrettyPrinter.companionTrait(decl: Declaration) { + protected fun PrettyPrinter.companionTrait(decl: Declaration, collector: MarshallersCollector) { if (decl.isConcrete) { println() + collector.addMarshaller(decl.namespace, decl.name) block("companion object : IMarshaller<${decl.name}>") { + "override val _type: KClass<${decl.name}> = ${decl.name}::class" + "override val id: RdId get() = RdId(${decl.name.getPlatformIndependentHash()})" @@ -543,6 +542,7 @@ open class Kotlin11Generator( } else if (decl.isOpen) { println() + collector.addMarshaller(decl.namespace, decl.name) block("companion object : IMarshaller<${decl.name}>, IAbstractDeclaration<${decl.name}>") { +"override val _type: KClass<${decl.name}> = ${decl.name}::class" +"override val id: RdId get() = RdId(${decl.name.getPlatformIndependentHash()})" @@ -563,7 +563,7 @@ open class Kotlin11Generator( println() block("companion object : ISerializersOwner") { println() - registerSerializersTrait(decl, decl.declaredTypes + unknowns(decl.declaredTypes)) + registerSerializersTrait(decl, decl.declaredTypes + unknowns(decl.declaredTypes), collector) println() println() createModelMethodTrait(decl) @@ -610,23 +610,32 @@ open class Kotlin11Generator( } } - protected fun PrettyPrinter.registerSerializersTrait(decl: Toplevel, types: List) { + protected fun PrettyPrinter.registerSerializersTrait(decl: Toplevel, types: List, collector: MarshallersCollector) { block("override fun registerSerializersCore(serializers: ISerializers) ") { var first = true - types.filter { !it.isAbstract }.filterIsInstance().println { + types.filter { !it.isAbstract }.filterIsInstance().forEach { if (it is Declaration && it.getSetting(Intrinsic) == null) { - if (first) { + if (first && collector.shouldGenerateRegistrations) { +"val classLoader = javaClass.classLoader" first = false } - "serializers.register(LazyCompanionMarshaller(RdId(${it.name.getPlatformIndependentHash()}), classLoader, \"${it.namespace}.${it.name}\"))" + + collector.addMarshaller(it.namespace, it.name) + if (collector.shouldGenerateRegistrations) { + println("serializers.register(LazyCompanionMarshaller(RdId(${it.name.getPlatformIndependentHash()}), classLoader, \"${it.namespace}.${it.name}\"))") + } } else { - "serializers.register(${it.serializerRef(decl, true)})" + println("serializers.register(${it.serializerRef(decl, true)})") } } if (decl is Root) { - decl.toplevels.println { it.sanitizedName(decl) + ".register(serializers)" } + decl.toplevels.forEach { + collector.addMarshaller(decl.namespace, decl.name) + if (collector.shouldGenerateRegistrations) { + println(it.sanitizedName(decl) + ".register(serializers)") + } + } } } }