cleaning PR after tests

codegen/core/src/main/scala/io/kalix/codegen/DescriptorSet.scala

@@ -5,16 +5,26 @@
 
 package io.kalix.codegen
 
-import com.google.protobuf.{ DescriptorProtos, Descriptors }
+import java.io.FileInputStream
+import java.io.FileNotFoundException
+import java.io.IOException
+import java.io.InputStream
+import java.util.logging.Level
+import java.util.logging.Logger
 
-import java.io.{ File, FileInputStream, FileNotFoundException, IOException }
-import java.util.logging.{ Level, Logger }
 import scala.jdk.CollectionConverters._
-import scala.util.{ Failure, Success, Using }
+import scala.util.Failure
+import scala.util.Success
+import scala.util.Using
+
 import com.google.protobuf.ExtensionRegistry
+import com.google.protobuf.DescriptorProtos
+import com.google.protobuf.Descriptors
 
 /**
  * Provides conveniences for reading and parsing Protobuf descriptor sets
+ *
+ * Copy from JVM SDK
  */
 object DescriptorSet {
 
@@ -37,37 +47,60 @@ object DescriptorSet {
  * @return
  * a collection of FileDescriptor objects or an error condition
  */
- @SuppressWarnings(Array("org.wartremover.warts.Throw"))
- def fileDescriptors(file: File): Either[CannotOpen, Iterable[Either[ReadFailure, Descriptors.FileDescriptor]]] =
- Using[FileInputStream, Either[CannotOpen, Iterable[Either[ReadFailure, Descriptors.FileDescriptor]]]](
- new FileInputStream(file)) { fis =>
- val registry = ExtensionRegistry.newInstance()
- registry.add(kalix.Annotations.codegen)
- registry.add(kalix.Annotations.service)
- registry.add(kalix.Annotations.file)
- registry.add(kalix.Annotations.method)
-
- Right(try {
- val descriptorProtos =
- DescriptorProtos.FileDescriptorSet.parseFrom(fis, registry).getFileList.asScala
-
- for (descriptorProto <- descriptorProtos)
- yield try Right(Descriptors.FileDescriptor.buildFrom(descriptorProto, Array.empty, true))
- catch {
- case e: Descriptors.DescriptorValidationException =>
- Left(CannotValidate(e))
- }
- } catch {
- case e: IOException =>
- List(Left(CannotRead(e)))
- })
- } match {
+ def fileDescriptors(
+ file: java.io.File): Either[CannotOpen, Either[ReadFailure, Iterable[Descriptors.FileDescriptor]]] =
+ Using[FileInputStream, Either[CannotOpen, Either[ReadFailure, Iterable[Descriptors.FileDescriptor]]]](
+ new FileInputStream(file)) { fis => descriptors(fis) } match {
  case Success(result) => result
  case Failure(e: FileNotFoundException) => Left(CannotOpen(e))
  case Failure(e) => throw e
  }
 
+ @SuppressWarnings(Array("org.wartremover.warts.Throw"))
+ def descriptors(is: InputStream): Either[CannotOpen, Either[ReadFailure, Iterable[Descriptors.FileDescriptor]]] = {
+ val registry = ExtensionRegistry.newInstance()
+ registry.add(kalix.Annotations.codegen)
+ registry.add(kalix.Annotations.service)
+ registry.add(kalix.Annotations.file)
+ registry.add(kalix.Annotations.method)
+
+ Right(try {
+ val descriptorProtos =
+ DescriptorProtos.FileDescriptorSet.parseFrom(is, registry).getFileList.asScala
+
+ val empty: Either[ReadFailure, Iterable[Descriptors.FileDescriptor]] =
+ Right(Array[Descriptors.FileDescriptor]())
+ descriptorProtos.foldLeft(empty)((acc, file) => accumulatedBuildFrom(acc, file))
+ } catch {
+ case e: IOException =>
+ Left(CannotRead(e))
+ })
+ }
+
  private val descriptorslogger = Logger.getLogger(classOf[Descriptors].getName)
  descriptorslogger.setLevel(Level.OFF); // Silence protobuf
 
+ /**
+ * This method accumulates `FileDescriptor`s to provide all the necessary dependencies for each call to
+ * FileDescriptor.buildFrom. Otherwise placeholders (mocked references) get created instead and these can't function
+ * as proper dependencies. Chiefly as imports.
+ *
+ * see allowUnknownDependencies per
+ * https://github.com/protocolbuffers/protobuf/blob/ae26a81918fa9e16f64ac27b5a2fb2b110b7aa1b/java/core/src/main/java/com/google/protobuf/Descriptors.java#L286
+ */
+ private def accumulatedBuildFrom(
+ reads: Either[ReadFailure, Iterable[Descriptors.FileDescriptor]],
+ file: DescriptorProtos.FileDescriptorProto): Either[ReadFailure, Iterable[Descriptors.FileDescriptor]] = {
+ reads match {
+ case Left(_) => reads
+ case Right(fileDescriptors) =>
+ try {
+ Right(fileDescriptors ++ List(Descriptors.FileDescriptor.buildFrom(file, fileDescriptors.toArray, true)))
+ } catch {
+ case e: Descriptors.DescriptorValidationException =>
+ Left(CannotValidate(e))
+ }
+ }
+ }
+
 }

codegen/core/src/test/scala/io/kalix/codegen/DescriptorSetSuite.scala

@@ -14,7 +14,7 @@ class DescriptorSetSuite extends munit.FunSuite {
  val descriptorFile = testFilesPath.resolve("descriptor-sets/hello-1.0-SNAPSHOT.protobin").toFile
  val result = DescriptorSet
  .fileDescriptors(descriptorFile)
- .flatMap(x => x.head.map(_.getServices.get(0).getFullName))
+ .flatMap(x => x.map(_.head.getServices.get(0).getFullName))
  assertEquals(result, Right("com.lightbend.MyServiceEntity"))
  }
 

codegen/js-gen-cli/src/main/scala/io/kalix/codegen/js/Cli.scala

@@ -82,7 +82,7 @@ object Cli {
  config.descriptorSetOutputDirectory.resolve(config.descriptorSetFileName).toFile
  if (protobufDescriptor.exists) {
  println("Inspecting proto file descriptor for Kalix code generation...")
- val _ = DescriptorSet2.fileDescriptors(protobufDescriptor) match {
+ val _ = DescriptorSet.fileDescriptors(protobufDescriptor) match {
  case Right(fileDescriptors) =>
  val model =
  ModelBuilder.introspectProtobufClasses(fileDescriptors match {
@@ -109,7 +109,7 @@ object Cli {
  println("Generated: " + absBaseDir.relativize(p.toAbsolutePath).toString)
  }
 
- case Left(DescriptorSet2.CannotOpen(e)) =>
+ case Left(DescriptorSet.CannotOpen(e)) =>
  System.err.println("There was a problem opening the protobuf descriptor file:")
  System.err.println(e.toString)
  sys.exit(1)

sdk/bin/download-protoc.js

@@ -7,10 +7,9 @@ const path = require('path');
 const fs = require('fs');
 const rimraf = require('rimraf');
 
-// https://github.com/protocolbuffers/protobuf/releases/download/v3.25.2/protoc-3.25.2-linux-x86_64.zip
 const downloadUrlPrefix =
  'https://github.com/protocolbuffers/protobuf/releases/download/v';
-const protocVersion = '25.2';
+const protocVersion = '3.20.1';
 function makeDownloadFile(platformArch) {
  return 'protoc-' + protocVersion + '-' + platformArch + '.zip';
 }