Prevent JSON from parsing malformed input

Kotlin · Jun 19, 2024 · 5931d15 · 5931d15
1 parent 0adbb10
commit 5931d15
Show file tree

Hide file tree

Showing 8 changed files with 61 additions and 243 deletions.
diff --git a/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/io/readJson.kt b/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/io/readJson.kt
@@ -215,6 +215,7 @@ internal fun fromJsonListAnyColumns(
                             v.longOrNull != null -> collector.add(v.long)
                             v.doubleOrNull != null -> collector.add(v.double)
                             v.floatOrNull != null -> collector.add(v.float)
+                            else -> error("Malformed JSON element ${v::class}: $v")
                         }
                     }
 
@@ -527,6 +528,7 @@ internal fun fromJsonListArrayAndValueColumns(
                                         v.longOrNull != null -> collector.add(v.long)
                                         v.doubleOrNull != null -> collector.add(v.double)
                                         v.floatOrNull != null -> collector.add(v.float)
+                                        else -> error("Malformed JSON element ${v::class}: $v")
                                     }
                                 }
 

diff --git a/core/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/CsvTests.kt b/core/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/CsvTests.kt
@@ -22,6 +22,7 @@ import org.jetbrains.kotlinx.dataframe.testResource
 import org.junit.Test
 import java.io.File
 import java.io.StringWriter
+import java.net.URL
 import java.util.Locale
 import kotlin.reflect.KClass
 import kotlin.reflect.typeOf
@@ -249,6 +250,14 @@ class CsvTests {
         producedFile.delete()
     }
 
+    @Test
+    fun `check integrity of example data`() {
+        val df = DataFrame.readCSV("../data/jetbrains_repositories.csv")
+        df.columnNames() shouldBe listOf("full_name", "html_url", "stargazers_count", "topics", "watchers")
+        df.columnTypes() shouldBe listOf(typeOf<String>(), typeOf<URL>(), typeOf<Int>(), typeOf<String>(), typeOf<Int>())
+        df shouldBe DataFrame.readCSV("../data/jetbrains repositories.csv")
+    }
+
     companion object {
         private val simpleCsv = testCsv("testCSV")
         private val csvWithFrenchLocale = testCsv("testCSVwithFrenchLocale")

diff --git a/core/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/json.kt b/core/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/json.kt
@@ -1,12 +1,15 @@
 package org.jetbrains.kotlinx.dataframe.io
 
 import io.kotest.assertions.throwables.shouldNotThrowAny
+import io.kotest.assertions.throwables.shouldThrow
 import io.kotest.matchers.collections.shouldBeIn
 import io.kotest.matchers.shouldBe
 import io.kotest.matchers.string.shouldContain
 import io.kotest.matchers.string.shouldNotContain
 import io.kotest.matchers.types.instanceOf
 import io.kotest.matchers.types.shouldBeInstanceOf
+import kotlinx.serialization.json.Json
+import kotlinx.serialization.json.JsonElement
 import kotlinx.serialization.json.boolean
 import kotlinx.serialization.json.int
 import kotlinx.serialization.json.jsonArray
@@ -42,6 +45,7 @@ import org.jetbrains.kotlinx.dataframe.impl.io.SerializationKeys.METADATA
 import org.jetbrains.kotlinx.dataframe.impl.io.SerializationKeys.NCOL
 import org.jetbrains.kotlinx.dataframe.impl.io.SerializationKeys.NROW
 import org.jetbrains.kotlinx.dataframe.impl.io.SerializationKeys.VERSION
+import org.jetbrains.kotlinx.dataframe.impl.io.readJson
 import org.jetbrains.kotlinx.dataframe.impl.nothingType
 import org.jetbrains.kotlinx.dataframe.io.JSON.TypeClashTactic.ANY_COLUMNS
 import org.jetbrains.kotlinx.dataframe.io.JSON.TypeClashTactic.ARRAY_AND_VALUE_COLUMNS
@@ -1077,4 +1081,13 @@ class JsonTests {
         val json = df.toJson()
         DataFrame.readJsonStr(json) shouldBe df
     }
+
+    @Test
+    fun `parse invalid literal`() {
+        // https://github.com/Kotlin/kotlinx.serialization/issues/2511
+        val json = Json.decodeFromString<JsonElement>("""[jetbrains, jetbrains-youtrack, youtrack, youtrack-api]""")
+        shouldThrow<IllegalStateException> {
+            readJson(json, emptyList())
+        }
+    }
 }
diff --git a/...taframe-gradle-plugin/src/test/kotlin/org/jetbrains/dataframe/gradle/DataFrameReadTest.kt b/...taframe-gradle-plugin/src/test/kotlin/org/jetbrains/dataframe/gradle/DataFrameReadTest.kt
@@ -33,7 +33,7 @@ class DataFrameReadTest {
     fun `file with invalid json`() {
         val temp = Files.createTempDirectory("").toFile()
         val invalidJson = File(temp, "test.json").also { it.writeText(".") }
-        shouldNotThrowAny {
+        shouldThrow<IllegalStateException> {
             DataFrame.read(invalidJson)
         }
     }

diff --git a/plugins/kotlin-dataframe/testData/box/read.fir.ir.txt b/plugins/kotlin-dataframe/testData/box/read.fir.ir.txt
diff --git a/plugins/kotlin-dataframe/testData/box/read.fir.txt b/plugins/kotlin-dataframe/testData/box/read.fir.txt
@@ -7,8 +7,8 @@ FILE: read.kt
                 @R|org/jetbrains/kotlinx/dataframe/annotations/Order|(order = Int(0)) public abstract val full_name: R|kotlin/String|
                     public get(): R|kotlin/String|
 
-                @R|org/jetbrains/kotlinx/dataframe/annotations/Order|(order = Int(3)) public abstract val topics: R|org/jetbrains/kotlinx/dataframe/DataFrame<<local>/Topics_391>|
-                    public get(): R|org/jetbrains/kotlinx/dataframe/DataFrame<<local>/Topics_391>|
+                @R|org/jetbrains/kotlinx/dataframe/annotations/Order|(order = Int(3)) public abstract val topics: R|kotlin/String|
+                    public get(): R|kotlin/String|
 
                 @R|org/jetbrains/kotlinx/dataframe/annotations/Order|(order = Int(4)) public abstract val watchers: R|kotlin/Int|
                     public get(): R|kotlin/Int|
@@ -30,11 +30,11 @@ FILE: read.kt
                 public final val R|org/jetbrains/kotlinx/dataframe/ColumnsContainer<<local>/Read_16I>|.full_name: R|org/jetbrains/kotlinx/dataframe/DataColumn<kotlin/String>|
                     public get(): R|org/jetbrains/kotlinx/dataframe/DataColumn<kotlin/String>|
 
-                public final val R|org/jetbrains/kotlinx/dataframe/DataRow<<local>/Read_16I>|.topics: R|org/jetbrains/kotlinx/dataframe/DataFrame<<local>/Topics_391>|
-                    public get(): R|org/jetbrains/kotlinx/dataframe/DataFrame<<local>/Topics_391>|
+                public final val R|org/jetbrains/kotlinx/dataframe/DataRow<<local>/Read_16I>|.topics: R|kotlin/String|
+                    public get(): R|kotlin/String|
 
-                public final val R|org/jetbrains/kotlinx/dataframe/ColumnsContainer<<local>/Read_16I>|.topics: R|org/jetbrains/kotlinx/dataframe/DataColumn<org/jetbrains/kotlinx/dataframe/DataFrame<<local>/Topics_391>>|
-                    public get(): R|org/jetbrains/kotlinx/dataframe/DataColumn<org/jetbrains/kotlinx/dataframe/DataFrame<<local>/Topics_391>>|
+                public final val R|org/jetbrains/kotlinx/dataframe/ColumnsContainer<<local>/Read_16I>|.topics: R|org/jetbrains/kotlinx/dataframe/DataColumn<kotlin/String>|
+                    public get(): R|org/jetbrains/kotlinx/dataframe/DataColumn<kotlin/String>|
 
                 public final val R|org/jetbrains/kotlinx/dataframe/DataRow<<local>/Read_16I>|.watchers: R|kotlin/Int|
                     public get(): R|kotlin/Int|
@@ -58,34 +58,11 @@ FILE: read.kt
 
             }
 
-            local abstract class Topics_391 : R|kotlin/Any| {
-                @R|org/jetbrains/kotlinx/dataframe/annotations/Order|(order = Int(0)) public abstract val value: R|kotlin/Double|
-                    public get(): R|kotlin/Double|
-
-                public constructor(): R|<local>/Topics_391|
-
-            }
-
-            local final class Scope1 : R|kotlin/Any| {
-                public final val R|org/jetbrains/kotlinx/dataframe/DataRow<<local>/Topics_391>|.value: R|kotlin/Double|
-                    public get(): R|kotlin/Double|
-
-                public final val R|org/jetbrains/kotlinx/dataframe/ColumnsContainer<<local>/Topics_391>|.value: R|org/jetbrains/kotlinx/dataframe/DataColumn<kotlin/Double>|
-                    public get(): R|org/jetbrains/kotlinx/dataframe/DataColumn<kotlin/Double>|
-
-                public constructor(): R|<local>/Scope1|
-
-            }
-
             local abstract class Read_16 : R|<local>/Read_16I| {
                 public abstract var scope0: R|<local>/Scope0|
                     public get(): R|<local>/Scope0|
                     public set(value: R|<local>/Scope0|): R|kotlin/Unit|
 
-                public abstract var scope1: R|<local>/Scope1|
-                    public get(): R|<local>/Scope1|
-                    public set(value: R|<local>/Scope1|): R|kotlin/Unit|
-
                 public constructor(): R|<local>/Read_16|
 
             }