now working with spark-connect and scala 2.13

Kotlin · Jun 18, 2024 · 71f115a · 71f115a
1 parent 726edfd
commit 71f115a
Show file tree

Hide file tree

Showing 12 changed files with 73 additions and 35 deletions.
diff --git a/buildSrc/src/main/kotlin/Versions.kt b/buildSrc/src/main/kotlin/Versions.kt
@@ -1,9 +1,12 @@
+import org.gradle.jvm.toolchain.JavaLanguageVersion
+
 object Versions : Dsl<Versions> {
     const val project = "2.0.0-SNAPSHOT"
     const val kotlinSparkApiGradlePlugin = "2.0.0-SNAPSHOT"
     const val groupID = "org.jetbrains.kotlinx.spark"
     const val kotlin = "2.0.0-RC3"
-    const val jvmTarget = "8"
+    const val jvmTarget = "1.8"
+    val jvmLanguageVersion = JavaLanguageVersion.of(8)
     const val jupyterJvmTarget = "8"
     inline val spark get() = System.getProperty("spark") as String
     inline val scala get() = System.getProperty("scala") as String

diff --git a/compiler-plugin/build.gradle.kts b/compiler-plugin/build.gradle.kts
@@ -1,3 +1,4 @@
+import org.jetbrains.kotlin.gradle.dsl.JvmTarget
 import org.jetbrains.kotlin.gradle.dsl.KotlinVersion
 import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
 
@@ -74,13 +75,17 @@ tasks.withType<KotlinCompile>().configureEach {
 
 kotlin {
     jvmToolchain {
-        languageVersion = JavaLanguageVersion.of(8)
+        languageVersion = Versions.jvmLanguageVersion
+    }
+    compilerOptions {
+        jvmTarget = JvmTarget.fromTarget(Versions.jvmTarget)
     }
 }
 java {
     toolchain {
         languageVersion = JavaLanguageVersion.of(8)
     }
+    targetCompatibility = JavaVersion.toVersion(Versions.jvmTarget)
 }
 
 val generateTests by tasks.creating(JavaExec::class) {

diff --git a/examples/build.gradle.kts b/examples/build.gradle.kts
@@ -1,4 +1,4 @@
-import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
+import org.jetbrains.kotlin.gradle.dsl.JvmTarget
 
 plugins {
     // Needs to be installed in the local maven repository or have the bootstrap jar on the classpath
@@ -49,8 +49,10 @@ dependencies {
 }
 
 kotlin {
-    jvmToolchain(8)
     jvmToolchain {
-        languageVersion = JavaLanguageVersion.of(Versions.jvmTarget)
+        languageVersion = Versions.jvmLanguageVersion
+    }
+    compilerOptions {
+        jvmTarget = JvmTarget.fromTarget(Versions.jvmTarget)
     }
 }
diff --git a/gradle-plugin/build.gradle.kts b/gradle-plugin/build.gradle.kts
@@ -1,5 +1,8 @@
 @file:Suppress("UnstableApiUsage")
 
+import org.jetbrains.kotlin.gradle.dsl.JvmTarget
+
+
 plugins {
     `java-gradle-plugin`
     kotlin
@@ -51,7 +54,10 @@ dependencies {
 
 kotlin {
     jvmToolchain {
-        languageVersion = JavaLanguageVersion.of(Versions.jvmTarget)
+        languageVersion = Versions.jvmLanguageVersion
+    }
+    compilerOptions {
+        jvmTarget = JvmTarget.fromTarget(Versions.jvmTarget)
     }
 }
 

diff --git a/gradle/bootstraps/compiler-plugin.jar b/gradle/bootstraps/compiler-plugin.jar
diff --git a/gradle/bootstraps/gradle-plugin.jar b/gradle/bootstraps/gradle-plugin.jar
diff --git a/kotlin-spark-api/build.gradle.kts b/kotlin-spark-api/build.gradle.kts
@@ -4,6 +4,7 @@ import com.igormaznitsa.jcp.gradle.JcpTask
 import com.vanniktech.maven.publish.JavadocJar.Dokka
 import com.vanniktech.maven.publish.KotlinJvm
 import org.jetbrains.dokka.gradle.AbstractDokkaLeafTask
+import org.jetbrains.kotlin.gradle.dsl.JvmTarget
 
 plugins {
     kotlin
@@ -66,6 +67,9 @@ dependencies {
             kafkaStreamsTestUtils,
             sparkMl,
         )
+
+        compileOnly(scalaLibrary)
+        testCompileOnly(scalaLibrary)
     }
 }
 
@@ -154,8 +158,11 @@ tasks.compileTestKotlin {
 }
 
 kotlin {
+    compilerOptions {
+        jvmTarget = JvmTarget.fromTarget(Versions.jvmTarget)
+    }
     jvmToolchain {
-        languageVersion = JavaLanguageVersion.of(Versions.jvmTarget)
+        languageVersion = Versions.jvmLanguageVersion
     }
 }
 

diff --git a/kotlin-spark-api/src/main/kotlin/org/jetbrains/kotlinx/spark/api/Encoding.kt b/kotlin-spark-api/src/main/kotlin/org/jetbrains/kotlinx/spark/api/Encoding.kt
@@ -76,6 +76,7 @@ import kotlin.reflect.jvm.javaGetter
 import kotlin.reflect.jvm.javaMethod
 import kotlin.reflect.jvm.jvmName
 import kotlin.reflect.typeOf
+import scala.collection.immutable.Seq as ImmutableSeq
 
 fun <T : Any> kotlinEncoderFor(
     kClass: KClass<T>,
@@ -683,7 +684,7 @@ object KotlinTypeInference : Serializable {
 
                 JavaBeanEncoder<Any>(
                     ClassTag.apply(jClass),
-                    fields.asScalaSeq(),
+                    fields.asScalaSeq() as ImmutableSeq<EncoderField>?,
                 )
             }
         }

diff --git a/scala-helpers/build.gradle.kts b/scala-helpers/build.gradle.kts
@@ -21,8 +21,7 @@ repositories {
 dependencies {
 
     Dependencies {
-        api(
-            scalaLibrary,
+        implementation(
             reflect,
         )
 
@@ -32,18 +31,20 @@ dependencies {
         implementation(
 //            sparkSql, not needed atm
         )
+        compileOnly(scalaLibrary)
+        testCompileOnly(scalaLibrary)
     }
 }
 
 java {
     toolchain {
         if (Versions.scalaCompat.toDouble() > 2.12) { // scala 2.12 will always target java 8
-            languageVersion = JavaLanguageVersion.of(Versions.jvmTarget)
-
+            languageVersion = Versions.jvmLanguageVersion
         } else if (Versions.jvmTarget == "1.8" || Versions.jvmTarget == "8") {
             languageVersion = JavaLanguageVersion.of(8)
         }
     }
+    targetCompatibility = JavaVersion.toVersion(Versions.jvmTarget)
 }
 
 tasks.withType<ScalaCompile> {
@@ -54,9 +55,12 @@ tasks.withType<ScalaCompile> {
     }
 }
 
-val scalaMainSources = sourceSets.main.get().scala.sourceDirectories
+val scalaMainSources =
+    sourceSets.main
+        .get()
+        .scala.sourceDirectories
 
-val preprocessMain by tasks.creating(JcpTask::class)  {
+val preprocessMain by tasks.creating(JcpTask::class) {
     sources = scalaMainSources
     clearTarget = true
     fileExtensions = listOf("scala")
@@ -99,4 +103,4 @@ mavenPublishing {
 val skipScalaOnlyDependent = System.getProperty("skipScalaOnlyDependent").toBoolean()
 tasks
     .filter { "publish" in it.name }
-    .forEach { it.onlyIf { !skipScalaOnlyDependent } }
+    .forEach { it.onlyIf { !skipScalaOnlyDependent } }
diff --git a/scala-tuples-in-kotlin/build.gradle.kts b/scala-tuples-in-kotlin/build.gradle.kts
@@ -3,9 +3,7 @@
 import com.vanniktech.maven.publish.JavadocJar.Dokka
 import com.vanniktech.maven.publish.KotlinJvm
 import org.jetbrains.dokka.gradle.AbstractDokkaLeafTask
-import org.jetbrains.dokka.gradle.DokkaTask
-import org.jetbrains.dokka.gradle.DokkaTaskPartial
-import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
+import org.jetbrains.kotlin.gradle.dsl.JvmTarget
 
 plugins {
     scala
@@ -30,24 +28,26 @@ dependencies {
     Dependencies {
         implementation(
             kotlinStdLib,
-            scalaLibrary,
         )
         testImplementation(
             kotest,
             atrium,
             kotlinTest,
         )
+        compileOnly(scalaLibrary)
+        testCompileOnly(scalaLibrary)
     }
 }
 
-
 kotlin {
     jvmToolchain {
-        languageVersion = JavaLanguageVersion.of(Versions.jvmTarget)
+        languageVersion = Versions.jvmLanguageVersion
+    }
+    compilerOptions {
+        jvmTarget = JvmTarget.fromTarget(Versions.jvmTarget)
     }
 }
 
-
 tasks.withType<AbstractDokkaLeafTask> {
     dokkaSourceSets {
         all {

diff --git a/spark-connect-examples/build.gradle.kts b/spark-connect-examples/build.gradle.kts
@@ -5,6 +5,7 @@ plugins {
     id("org.jetbrains.kotlinx.spark.api")
     kotlin("jvm")
     application
+    shadow
 }
 
 // run with `./gradlew run`
@@ -44,21 +45,21 @@ dependencies {
         implementation(sparkConnectClient)
 
         implementation(kotlinDateTime)
+
+        compileOnly(scalaLibrary)
     }
 }
 
-// spark-connect seems to work well with java 17 as client and java 1.8 as server
-// also set gradle and your project sdk to java 17
 kotlin {
     jvmToolchain {
-        languageVersion = JavaLanguageVersion.of(17)
+        languageVersion = JavaLanguageVersion.of(17)//Versions.jvmLanguageVersion
     }
     compilerOptions {
-        jvmTarget = JvmTarget.JVM_17
+        jvmTarget = JvmTarget.fromTarget(Versions.jvmTarget)
     }
 }
 
 tasks.withType<JavaCompile> {
-    sourceCompatibility = JavaVersion.VERSION_17.toString()
-    targetCompatibility = JavaVersion.VERSION_17.toString()
+    sourceCompatibility = JavaVersion.VERSION_1_8.toString()
+    targetCompatibility = JavaVersion.VERSION_1_8.toString()
 }
diff --git a/spark-connect-examples/src/main/kotlin/org/jetbrains/kotlinx/spark/examples/Main.kt b/spark-connect-examples/src/main/kotlin/org/jetbrains/kotlinx/spark/examples/Main.kt
@@ -5,30 +5,39 @@ import org.jetbrains.kotlinx.spark.api.plugin.annotations.Sparkify
 import org.jetbrains.kotlinx.spark.api.showDS
 import org.jetbrains.kotlinx.spark.api.toList
 import org.jetbrains.kotlinx.spark.api.tuples.X
-import org.jetbrains.kotlinx.spark.api.tuples.t
 import org.jetbrains.kotlinx.spark.api.withSparkConnect
 import scala.Tuple2
 import java.time.LocalDate
 
-// run with `./gradlew run` or set VM options: "--add-opens=java.base/java.nio=ALL-UNNAMED" in the IDE
+// run with `./gradlew runShadow` or set VM options: "--add-opens=java.base/java.nio=ALL-UNNAMED" in the IDE
 fun main() =
     withSparkConnect("sc://localhost") {
         val classFinder = REPLClassDirMonitor("/mnt/data/Projects/kotlin-spark-api/spark-connect-examples/build/classes")
         spark.registerClassFinder(classFinder)
-        spark.addArtifact("/mnt/data/Projects/kotlin-spark-api/spark-connect-examples/build/libs/spark-connect-examples-2.0.0-SNAPSHOT.jar")
+        // make jar first, preferably a fat jar with shadow, but be careful it doesn't contain scala depencencies
+        spark.addArtifact("/mnt/data/Projects/kotlin-spark-api/spark-connect-examples/build/libs/spark-connect-examples-2.0.0-SNAPSHOT-all.jar")
+
 
         val data =
             listOf(
                 Person("Alice", 25, LocalDate.of(1996, 1, 1), "Alice" X Address("1 Main St", "Springfield", "IL", 62701)),
                 Person("Bob", 30, LocalDate.of(1991, 1, 1), "Bob" X Address("2 Main St", "Springfield", "IL", 62701)),
-                Person("Charlie", 35, LocalDate.of(1986, 1, 1), "Charlie" X Address("3 Main St", "Springfield", "IL", 62701)),
+                Person(
+                    "Charlie",
+                    35,
+                    LocalDate.of(1986, 1, 1),
+                    "Charlie" X Address("3 Main St", "Springfield", "IL", 62701),
+                ),
             )
 
         val ds = data.toDS().showDS()
 
-        ds.toList<Person>().forEach {
-            println(it)
-        }
+        ds
+            .filter { it.age > 26 }
+            .toList<Person>()
+            .forEach {
+                println(it)
+            }
     }
 
 @Sparkify