Support saving the list of marshallers during generation to a file

Support passing MarshallersProvider to avoid greedy registration of all serializers on the startup, and just search for them when they are needed. It also solves the `Maybe you forgot to invoke 'register()' method` problem

rd-kt/rd-framework/src/main/kotlin/com/jetbrains/rd/framework/MarshallersProvider.kt

@@ -0,0 +1,44 @@
+package com.jetbrains.rd.framework
+
+import com.jetbrains.rd.util.error
+import com.jetbrains.rd.util.getLogger
+import java.io.InputStream
+
+interface MarshallersProvider {
+    object Dummy : MarshallersProvider {
+        override fun getMarshaller(id: RdId): IMarshaller<*>? = null
+    }
+
+    companion object {
+        fun extractMarshallers(
+            stream: InputStream,
+            classsLoader: ClassLoader
+        ): List<IMarshaller<*>> = stream.reader().useLines { lines: Sequence<String> ->
+            lines.map<String, List<String>> { it.split(":") }.map<List<String>, LazyCompanionMarshaller<Any>> {
+                LazyCompanionMarshaller<Any>(RdId(it.first().toLong()), classsLoader, it.last())
+            }.toList()
+        }
+    }
+
+    fun getMarshaller(id: RdId): IMarshaller<*>?
+}
+
+class AggregatedMarshallersProvider(val providers: Sequence<MarshallersProvider>) : MarshallersProvider {
+    override fun getMarshaller(id: RdId): IMarshaller<*>? {
+        return providers.mapNotNull { it.getMarshaller(id) }.firstOrNull()
+    }
+}
+
+abstract class MarhallersProviderFromResourcesBase(val resourceName: String) : MarshallersProvider {
+    private val map: Map<RdId, IMarshaller<*>> by lazy {
+        val classLoader = javaClass.classLoader
+        val resource = classLoader.getResourceAsStream(resourceName) ?: run {
+            getLogger(this::class).error { "$resourceName is not found" }
+            return@lazy emptyMap()
+        }
+
+        MarshallersProvider.extractMarshallers(resource, classsLoader = classLoader).associateBy { it.id }
+    }
+
+    override fun getMarshaller(id: RdId): IMarshaller<*>? = map[id]
+}

rd-kt/rd-framework/src/main/kotlin/com/jetbrains/rd/framework/Serializers.kt

@@ -5,9 +5,7 @@ import com.jetbrains.rd.framework.impl.RdSecureString
 import com.jetbrains.rd.util.*
 import com.jetbrains.rd.util.hash.getPlatformIndependentHash
 import com.jetbrains.rd.util.lifetime.Lifetime
-import java.util.concurrent.TimeUnit
 import kotlin.reflect.KClass
-import kotlin.reflect.jvm.jvmName
 import kotlin.time.Duration
 import kotlin.time.DurationUnit
 import kotlin.time.toDuration
@@ -16,12 +14,14 @@ private const val notRegisteredErrorMessage = "Maybe you forgot to invoke 'regis
         "Usually it should be done automatically during 'bind()' invocation but in complex cases you should do it manually."
 
 
-@Suppress("UNCHECKED_CAST")
-class Serializers : ISerializers {
-
+class Serializers(private val provider: MarshallersProvider) : ISerializers {
     companion object {
         private val backgroundRegistrar =  createBackgroundScheduler(Lifetime.Eternal, "SerializersBackgroundRegistrar")
     }
+
+    @Deprecated("Use an overload with MarshallersProvider")
+    constructor() : this(MarshallersProvider.Dummy)
+
     override fun registerSerializersOwnerOnce(serializersOwner: ISerializersOwner) {
         backgroundRegistrar.invokeOrQueue {
             val key = serializersOwner::class
@@ -54,7 +54,7 @@ class Serializers : ISerializers {
         val id = serializer.id
         val existing = marshallers[id]
         if (existing != null) {
-            require(existing.fqn == serializer.fqn) { "Can't register ${serializer.fqn} with id: $id, already registered: ${serializer.fqn}" }
+            assertSerializersAreTheSame(existing, serializer, id)
         }  else {
             Protocol.initializationLogger.trace { "Registering type ${serializer.fqn}, id = $id" }
             marshallers[id] = serializer
@@ -63,6 +63,10 @@ class Serializers : ISerializers {
         }
     }
 
+    private fun assertSerializersAreTheSame(existing: IMarshaller<*>, new: IMarshaller<*>, id: RdId) {
+        require(existing.fqn == new.fqn) { "Can't register ${new.fqn} with id: $id, already registered: ${new.fqn}" }
+    }
+
     override fun get(id: RdId): IMarshaller<*>? {
         return marshallers[id]
     }
@@ -75,7 +79,14 @@ class Serializers : ISerializers {
         val size = stream.readInt()
         stream.checkAvailable(size)
 
-        val reader = marshallers[id]
+        val reader = marshallers[id] ?: run {
+            provider.getMarshaller(id)?.let { newMarshaller ->
+                marshallers.putIfAbsent(id, newMarshaller)?.also { existing ->
+                    assertSerializersAreTheSame(existing, newMarshaller, id)
+                } ?: newMarshaller
+            }
+        }
+
         if (reader == null) {
             if (abstractDeclaration == null) {
                 throw IllegalStateException("Can't find reader by id: $id. $notRegisteredErrorMessage")
@@ -103,7 +114,7 @@ class Serializers : ISerializers {
     private fun <T : Any> getWriter(clazz: KClass<out T>): IMarshaller<T> {
         val marshaller = writers.getOrPut(clazz) {
             val id = RdId(clazz.simpleName.getPlatformIndependentHash())
-            marshallers[id] ?: cantFindWriter(clazz)
+            marshallers[id] ?: provider.getMarshaller(id) ?: cantFindWriter(clazz)
         }
 
         return marshaller as? IMarshaller<T> ?: cantFindWriter(clazz)

rd-kt/rd-gen/src/gradlePlugin/kotlin/com/jetbrains/rd/generator/gradle/GradleGenerationSpec.kt

@@ -7,8 +7,9 @@ class GradleGenerationSpec {
     var namespace = ""
     var directory = ""
     var generatedFileSuffix = ".Generated"
+    var marshallersFile: String? = ""
 
     override fun toString(): String {
-        return "$language||$transform||$root||$namespace||$directory||$generatedFileSuffix"
+        return "$language||$transform||$root||$namespace||$directory||$generatedFileSuffix||${marshallersFile}"
     }
 }

rd-kt/rd-gen/src/main/kotlin/com/jetbrains/rd/generator/nova/GenerationSpec.kt

@@ -11,18 +11,21 @@ data class GenerationSpec(
     var root: String = "",
     var namespace: String = "",
     var directory: String = "",
-    var generatedFileSuffix: String = ".Generated"
+    var generatedFileSuffix: String = ".Generated",
+    var marshallersFile: String? = null,
 ) {
     companion object {
         fun loadFrom(file: File): List<GenerationSpec> {
             val result = mutableListOf<GenerationSpec>()
             val lines = file.readLines(Charsets.UTF_8)
             for (line in lines) {
                 val parts = line.split("||")
-                result.add(GenerationSpec(parts[0], parts[1], parts[2], parts[3], parts[4], parts[5]))
+                result.add(GenerationSpec(parts[0], parts[1], parts[2], parts[3], parts[4], parts[5], parts[6].nullIfEmpty()))
             }
             return result
         }
+
+        fun String.nullIfEmpty(): String? = if (this.isBlank()) null else this
     }
 
     fun toGeneratorAndRoot(availableRoots: List<Root>): IGeneratorAndRoot {
@@ -34,8 +37,8 @@ data class GenerationSpec(
             else -> throw GeneratorException("Unknown flow transform type ${transform}, use 'asis', 'reversed' or 'symmetric'")
         }
         val generator = when (language) {
-            "kotlin" -> Kotlin11Generator(flowTransform, namespace, File(directory), generatedFileSuffix)
-            "csharp" -> CSharp50Generator(flowTransform, namespace, File(directory), generatedFileSuffix)
+            "kotlin" -> Kotlin11Generator(flowTransform, namespace, File(directory), generatedFileSuffix, marhsallersFile = marshallersFile?.let { File(it) })
+            "csharp" -> CSharp50Generator(flowTransform, namespace, File(directory), generatedFileSuffix) // todo support for C#
             "cpp" -> Cpp17Generator(flowTransform, namespace, File(directory), generatedFileSuffix)
             else -> throw GeneratorException("Unknown language $language, use 'kotlin' or 'csharp' or 'cpp'")
         }

rd-kt/rd-gen/src/main/kotlin/com/jetbrains/rd/generator/nova/Generators.kt

@@ -44,7 +44,7 @@ fun fail(msg: String) : Nothing { throw GeneratorException(msg) }
 /**
  * Base class for generators to deduplicate common logic
  */
-abstract class GeneratorBase(protected open val flowTransform: FlowTransform, protected val generatedFileSuffix: String) : IGenerator {
+abstract class GeneratorBase(protected open val flowTransform: FlowTransform, protected val generatedFileSuffix: String, val marhsallersFile: File? = null,) : IGenerator {
     object AllowDeconstruct: ISetting<Unit, Declaration>
 
     /**
@@ -53,16 +53,33 @@ abstract class GeneratorBase(protected open val flowTransform: FlowTransform, pr
     object AcceptsGenerator: ISetting<(IGenerator) -> Boolean, Toplevel>
 
 
-    protected abstract fun realGenerate(toplevels: List<Toplevel>)
+    protected abstract fun realGenerate(toplevels: List<Toplevel>, collector: MarshallersCollector)
 
     override fun generate(toplevels: List<Toplevel>) {
         val preparedToplevels = toplevels
             .filter { it.getSetting(AcceptsGenerator)?.invoke(this) ?: true }
             .sortedBy { it.name }
 
-        realGenerate(preparedToplevels)
+        withContext {
+            realGenerate(preparedToplevels, it)
+        }
     }
 
+    private fun withContext(action: (MarshallersCollector) -> Unit) {
+        val file = marhsallersFile
+        if (file != null) {
+            val context = RealMarshallersCollector(file)
+            try {
+                action(context)
+            }finally {
+                context.close()
+            }
+        } else {
+            action(DisabledMarshallersCollector)
+        }
+    }
+
+
     protected open fun unknowns(declaredTypes: Iterable<Declaration>): Collection<Declaration> {
         return declaredTypes.mapNotNull {
             val unknown: Declaration? = unknown(it)

rd-kt/rd-gen/src/main/kotlin/com/jetbrains/rd/generator/nova/MarshallersCollector.kt

@@ -0,0 +1,39 @@
+package com.jetbrains.rd.generator.nova
+
+import com.jetbrains.rd.util.hash.getPlatformIndependentHash
+import java.io.File
+
+
+interface MarshallersCollector {
+    val shouldGenerateRegistrations: Boolean
+
+    fun addMarshaller(namespace: String, name: String)
+}
+
+object DisabledMarshallersCollector : MarshallersCollector {
+    override val shouldGenerateRegistrations: Boolean
+        get() = true
+
+    override fun addMarshaller(namespace: String, name: String) {
+    }
+}
+
+class RealMarshallersCollector(val marshallersFile: File) : MarshallersCollector {
+    private val marshallers = mutableSetOf<String>()
+
+    override val shouldGenerateRegistrations: Boolean
+        get() = false // We may want to add a separate setting here, but for now just disable it
+
+    override fun addMarshaller(namespace: String, name: String) {
+        marshallers.add("${name.getPlatformIndependentHash()}:${namespace}.${name}")
+    }
+
+    fun close() {
+        marshallersFile.parentFile.mkdirs()
+        marshallersFile.writer().use { writer ->
+            marshallers.sorted().forEach {
+                writer.append(it).append("\n")
+            }
+        }
+    }
+}

rd-kt/rd-gen/src/main/kotlin/com/jetbrains/rd/generator/nova/cpp/Cpp17Generator.kt

@@ -902,7 +902,7 @@ open class Cpp17Generator(
         +"//------------------------------------------------------------------------------"
     }
 
-    override fun realGenerate(toplevels: List<Toplevel>) {
+    override fun realGenerate(toplevels: List<Toplevel>, collector: MarshallersCollector) {
         if (toplevels.isEmpty()) return
 
         val root = toplevels.first().root

rd-kt/rd-gen/src/main/kotlin/com/jetbrains/rd/generator/nova/csharp/CSharp50Generator.kt

@@ -360,7 +360,7 @@ open class CSharp50Generator(
 
     //generation
 
-    override fun realGenerate(toplevels: List<Toplevel>) {
+    override fun realGenerate(toplevels: List<Toplevel>, collector: MarshallersCollector) {
         toplevels.forEach { tl ->
             tl.fsPath.bufferedWriter().use { writer ->
                 PrettyPrinter().apply {