From bf89d3c1b65f31e12ae4ccbcbee2f91195a23f63 Mon Sep 17 00:00:00 2001 From: UnknownJoe796 Date: Fri, 13 Dec 2024 11:30:23 -0700 Subject: [PATCH] Support for encoding/decoding a sequence of values (#20) * Support for streaming via Reader and Appendable * Handle Microsoft Excel's insistence on using a byte order marker * Cleaning up new unit tests for FetchSourceTest * Removed commented debugging println's for FetchSource * Sequence encoding and decoding * Streaming serialization bug fix * Cleanup formatting * Refactor asynchronous API to `CsvRecordReader` and `CsvRecordWriter` * Fix blocking initialization of readers --------- Co-authored-by: Sven Obser --- gradle/libs.versions.toml | 2 + library/build.gradle.kts | 1 + .../serialization/csv/CsvRecordReader.kt | 48 ++++ .../serialization/csv/CsvRecordWriter.kt | 34 +++ .../serialization/csv/decode/CsvReader.kt | 17 +- .../serialization/csv/decode/FetchSource.kt | 24 +- .../serialization/csv/encode/CsvWriter.kt | 12 +- .../serialization/csv/example/ExampleTest.kt | 246 +++++++++++++----- 8 files changed, 304 insertions(+), 80 deletions(-) create mode 100644 library/src/main/kotlin/kotlinx/serialization/csv/CsvRecordReader.kt create mode 100644 library/src/main/kotlin/kotlinx/serialization/csv/CsvRecordWriter.kt diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 54b9f74..b4e1b47 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -2,6 +2,7 @@ dokka = "1.9.20" junit-jupiter = "5.11.0" kotlin = "2.1.0" +kotlinx-coroutines = "1.9.0" kotlinx-serialization-core = "1.7.3" nexus-publish = "0.4.0" nexus-staging = "0.30.0" @@ -9,6 +10,7 @@ researchgate-release = "3.0.2" [libraries] junit-jupiter = { module = "org.junit.jupiter:junit-jupiter", version.ref = "junit-jupiter" } +kotlinx-coroutines-test = { module = "org.jetbrains.kotlinx:kotlinx-coroutines-test", version.ref = "kotlinx-coroutines" } kotlinx-serialization-core = { module = "org.jetbrains.kotlinx:kotlinx-serialization-core", version.ref = "kotlinx-serialization-core" } [plugins] diff --git a/library/build.gradle.kts b/library/build.gradle.kts index 513d606..e8a15e8 100644 --- a/library/build.gradle.kts +++ b/library/build.gradle.kts @@ -15,6 +15,7 @@ dependencies { testImplementation(kotlin("test-junit5")) testImplementation(libs.junit.jupiter) + testImplementation(libs.kotlinx.coroutines.test) } kotlin { diff --git a/library/src/main/kotlin/kotlinx/serialization/csv/CsvRecordReader.kt b/library/src/main/kotlin/kotlinx/serialization/csv/CsvRecordReader.kt new file mode 100644 index 0000000..c5965cd --- /dev/null +++ b/library/src/main/kotlin/kotlinx/serialization/csv/CsvRecordReader.kt @@ -0,0 +1,48 @@ +package kotlinx.serialization.csv + +import kotlinx.serialization.ExperimentalSerializationApi +import kotlinx.serialization.KSerializer +import kotlinx.serialization.builtins.ListSerializer +import kotlinx.serialization.csv.decode.CsvReader +import kotlinx.serialization.csv.decode.FetchSource +import kotlinx.serialization.csv.decode.RecordListCsvDecoder +import kotlinx.serialization.encoding.CompositeDecoder.Companion.DECODE_DONE +import java.io.Reader + +/** + * Record reader that allows reading CSV line-by-line. + */ +interface CsvRecordReader : Iterator { + /** + * Read next record + */ + fun read(): T? = if (hasNext()) next() else null +} + +/** + * Parse CSV line-by-line from the given [input]. + * + * @param deserializer The deserializer used to parse the given CSV string. + * @param input The CSV reader to parse. This function *does not close the reader*. + */ +@ExperimentalSerializationApi +fun Csv.recordReader(deserializer: KSerializer, input: Reader): CsvRecordReader { + val decoder = RecordListCsvDecoder( + csv = this, + reader = CsvReader(FetchSource(input), config) + ) + val listDescriptor = ListSerializer(deserializer).descriptor + var previousValue: T? = null + + return object : CsvRecordReader { + override fun hasNext(): Boolean = + decoder.decodeElementIndex(listDescriptor) != DECODE_DONE + + override fun next(): T { + val index = decoder.decodeElementIndex(listDescriptor) + return decoder.decodeSerializableElement(listDescriptor, index, deserializer, previousValue).also { + previousValue = it + } + } + } +} diff --git a/library/src/main/kotlin/kotlinx/serialization/csv/CsvRecordWriter.kt b/library/src/main/kotlin/kotlinx/serialization/csv/CsvRecordWriter.kt new file mode 100644 index 0000000..1663aff --- /dev/null +++ b/library/src/main/kotlin/kotlinx/serialization/csv/CsvRecordWriter.kt @@ -0,0 +1,34 @@ +package kotlinx.serialization.csv + +import kotlinx.serialization.ExperimentalSerializationApi +import kotlinx.serialization.KSerializer +import kotlinx.serialization.csv.encode.CsvWriter +import kotlinx.serialization.csv.encode.RecordListCsvEncoder + +/** + * Record writer that allows writing CSV line by line. + */ +fun interface CsvRecordWriter { + /** + * Write next record. + */ + fun write(record: T) +} + +/** + * Create [CsvRecordWriter] that allows writing CSV line-by-line. + * + * @param serializer The serializer used to serialize the given object. + * @param output The output where the CSV will be written. + */ +@ExperimentalSerializationApi +fun Csv.recordWriter(serializer: KSerializer, output: Appendable): CsvRecordWriter { + val encoder = RecordListCsvEncoder( + csv = this, + writer = CsvWriter(output, config) + ) + + return CsvRecordWriter { + encoder.encodeSerializableValue(serializer, it) + } +} diff --git a/library/src/main/kotlin/kotlinx/serialization/csv/decode/CsvReader.kt b/library/src/main/kotlin/kotlinx/serialization/csv/decode/CsvReader.kt index 6eb1b79..88df669 100644 --- a/library/src/main/kotlin/kotlinx/serialization/csv/decode/CsvReader.kt +++ b/library/src/main/kotlin/kotlinx/serialization/csv/decode/CsvReader.kt @@ -5,7 +5,17 @@ import kotlinx.serialization.csv.config.CsvConfig /** * Reader that parses CSV input. */ -internal class CsvReader(private val source: Source, private val config: CsvConfig) { +internal class CsvReader(source: Source, private val config: CsvConfig) { + + private val source: Source by lazy { + source.also { + // Skip Microsoft Excel's byte order marker, should it appear. + // This has to happen lazily to avoid blocking read calls during the initialization of the CsvReader. + if (source.peek() == '\uFEFF') { + source.read() + } + } + } val offset get() = source.offset @@ -21,11 +31,6 @@ internal class CsvReader(private val source: Source, private val config: CsvConf private var marks = arrayListOf() - init { - // Skip Microsoft Excel's byte order marker, should it appear: - read("\uFEFF") - } - /** * Read value in the next column. */ diff --git a/library/src/main/kotlin/kotlinx/serialization/csv/decode/FetchSource.kt b/library/src/main/kotlin/kotlinx/serialization/csv/decode/FetchSource.kt index 60306d7..bf61094 100644 --- a/library/src/main/kotlin/kotlinx/serialization/csv/decode/FetchSource.kt +++ b/library/src/main/kotlin/kotlinx/serialization/csv/decode/FetchSource.kt @@ -1,5 +1,6 @@ package kotlinx.serialization.csv.decode +import java.io.EOFException import java.io.Reader internal class FetchSource( @@ -20,18 +21,27 @@ internal class FetchSource( override var offset: Int = 0 private set - private var next: Char? = getChar() + private var queue = ArrayList(2048) + private var marks = ArrayList(2048) + private var queueOffset = 0 + + private var next: Char? = null + get() { + if (field == null && nextPosition == 0) { + // Reading first char has to happen lazily to avoid blocking read calls + // during the initialization of the FetchSource. + field = getChar() + } + return field + } + private fun nextChar(): Char { - val n = next ?: throw IllegalStateException("Out of characters") + val nextChar = next ?: throw EOFException("No more characters to read.") next = getChar() nextPosition++ - return n + return nextChar } - private var queue = ArrayList(2048) - private var marks = ArrayList(2048) - private var queueOffset = 0 - override fun canRead(): Boolean = offset <= nextPosition override fun read(): Char? { diff --git a/library/src/main/kotlin/kotlinx/serialization/csv/encode/CsvWriter.kt b/library/src/main/kotlin/kotlinx/serialization/csv/encode/CsvWriter.kt index cf231b0..97e502d 100644 --- a/library/src/main/kotlin/kotlinx/serialization/csv/encode/CsvWriter.kt +++ b/library/src/main/kotlin/kotlinx/serialization/csv/encode/CsvWriter.kt @@ -9,7 +9,7 @@ import kotlinx.serialization.csv.config.QuoteMode * To write one CSV record, call [beginRecord], followed by multiple calls to [printColumn] and * finally call [endRecord] to finish the record. */ -internal class CsvWriter(private val sb: Appendable, private val config: CsvConfig) { +internal class CsvWriter(private val output: Appendable, private val config: CsvConfig) { var isFirstRecord = true private var isFirstColumn = true @@ -20,7 +20,7 @@ internal class CsvWriter(private val sb: Appendable, private val config: CsvConf */ fun beginRecord() { if (!isFirstRecord) { - sb.append(config.recordSeparator) + output.append(config.recordSeparator) } } @@ -64,19 +64,19 @@ internal class CsvWriter(private val sb: Appendable, private val config: CsvConf escapeCharacters = "$escapeChar$delimiter$quoteChar$recordSeparator", escapeChar = escapeChar ) - sb.append(escapedValue) + output.append(escapedValue) } else if (mode == WriteMode.QUOTED || mode == WriteMode.ESCAPED) { val escapedValue = value.replace("$quoteChar", "$quoteChar$quoteChar") - sb.append(quoteChar).append(escapedValue).append(quoteChar) + output.append(quoteChar).append(escapedValue).append(quoteChar) } else { - sb.append(value) + output.append(value) } } /** End the current column (which writes the column delimiter). */ private fun nextColumn() { if (!isFirstColumn) { - sb.append(config.delimiter) + output.append(config.delimiter) } isFirstColumn = false } diff --git a/library/src/test/kotlin/kotlinx/serialization/csv/example/ExampleTest.kt b/library/src/test/kotlin/kotlinx/serialization/csv/example/ExampleTest.kt index c2efc9e..ac87f71 100644 --- a/library/src/test/kotlin/kotlinx/serialization/csv/example/ExampleTest.kt +++ b/library/src/test/kotlin/kotlinx/serialization/csv/example/ExampleTest.kt @@ -1,19 +1,33 @@ package kotlinx.serialization.csv.example +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.async +import kotlinx.coroutines.delay +import kotlinx.coroutines.test.runTest import kotlinx.serialization.ExperimentalSerializationApi import kotlinx.serialization.builtins.ListSerializer import kotlinx.serialization.builtins.nullable import kotlinx.serialization.csv.Csv -import kotlinx.serialization.csv.example.Feature.* +import kotlinx.serialization.csv.example.Feature.AUTOMATIC +import kotlinx.serialization.csv.example.Feature.ELECTRIC +import kotlinx.serialization.csv.example.Feature.HEATED_SEATS +import kotlinx.serialization.csv.example.Feature.NAVIGATION_SYSTEM +import kotlinx.serialization.csv.example.Feature.XENON import kotlinx.serialization.csv.example.Tire.Axis.FRONT import kotlinx.serialization.csv.example.Tire.Axis.REAR import kotlinx.serialization.csv.example.Tire.Side.LEFT import kotlinx.serialization.csv.example.Tire.Side.RIGHT +import kotlinx.serialization.csv.recordReader +import kotlinx.serialization.csv.recordWriter import kotlinx.serialization.modules.SerializersModule import kotlinx.serialization.test.assertEncodeAndDecode +import java.io.PipedReader +import java.io.PipedWriter import java.time.LocalDateTime -import java.util.* +import java.util.UUID import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.time.Duration.Companion.milliseconds /** * Test complex [LocationRecord]. @@ -27,53 +41,54 @@ class ExampleTest { // Vehicles private val tesla = Vehicle( - UUID.fromString("f9682dcb-30f7-4e88-915e-60e3b2758da7"), - VehicleType.CAR, - "Tesla" + uuid = UUID.fromString("f9682dcb-30f7-4e88-915e-60e3b2758da7"), + type = VehicleType.CAR, + brand = "Tesla", ) private val porsche = Vehicle( - UUID.fromString("5e1afd88-97a2-4373-a83c-44a49c552abd"), - VehicleType.CAR, - "Porsche" + uuid = UUID.fromString("5e1afd88-97a2-4373-a83c-44a49c552abd"), + type = VehicleType.CAR, + brand = "Porsche", ) private val harley = Vehicle( - UUID.fromString("c038c27b-a3fd-4e35-b6ac-ab06d747e16c"), - VehicleType.BIKE, - "Harley" + uuid = UUID.fromString("c038c27b-a3fd-4e35-b6ac-ab06d747e16c"), + type = VehicleType.BIKE, + brand = "Harley", ) @Test fun testLocationRecords() = Csv { hasHeaderRecord = true }.assertEncodeAndDecode( - """|id,date,position.latitude,position.longitude,driver.id,driver.foreName,driver.lastName,driver.birthday,vehicle.uuid,vehicle.type,vehicle.brand,vehicleData.speed,vehicleData.consumption,vehicleData.consumption.Combustion.consumptionLiterPer100Km,vehicleData.consumption.Electric.consumptionKWhPer100Km - |0,2020-02-01T13:33:00,0.0,0.0,12,Jon,Smith,,f9682dcb-30f7-4e88-915e-60e3b2758da7,CAR,Tesla,,Unknown,, - |1,2020-02-01T13:37:00,0.1,0.1,12,Jon,Smith,,f9682dcb-30f7-4e88-915e-60e3b2758da7,CAR,Tesla,27.7778,Electric,,18.1 - |9000,2020-02-05T07:59:00,48.137154,11.576124,42,Jane,Doe,1581602631744,c038c27b-a3fd-4e35-b6ac-ab06d747e16c,MOTORBIKE,Harley,20.0,Combustion,7.9, + expected = """ + |id,date,position.latitude,position.longitude,driver.id,driver.foreName,driver.lastName,driver.birthday,vehicle.uuid,vehicle.type,vehicle.brand,vehicleData.speed,vehicleData.consumption,vehicleData.consumption.Combustion.consumptionLiterPer100Km,vehicleData.consumption.Electric.consumptionKWhPer100Km + |0,2020-02-01T13:33:00,0.0,0.0,12,Jon,Smith,,f9682dcb-30f7-4e88-915e-60e3b2758da7,CAR,Tesla,,Unknown,, + |1,2020-02-01T13:37:00,0.1,0.1,12,Jon,Smith,,f9682dcb-30f7-4e88-915e-60e3b2758da7,CAR,Tesla,27.7778,Electric,,18.1 + |9000,2020-02-05T07:59:00,48.137154,11.576124,42,Jane,Doe,1581602631744,c038c27b-a3fd-4e35-b6ac-ab06d747e16c,MOTORBIKE,Harley,20.0,Combustion,7.9, """.trimMargin(), - listOf( + original = listOf( LocationRecord( - 0, LocalDateTime.of(2020, 2, 1, 13, 33), - Position(0.0, 0.0), - jonSmith, tesla, - VehicleData(null, Consumption.Unknown) + id = 0, date = LocalDateTime.of(2020, 2, 1, 13, 33), + position = Position(0.0, 0.0), + driver = jonSmith, vehicle = tesla, + vehicleData = VehicleData(null, Consumption.Unknown), ), LocationRecord( - 1, LocalDateTime.of(2020, 2, 1, 13, 37), - Position(0.1, 0.1), - jonSmith, tesla, - VehicleData(27.7778, Consumption.Electric(18.1)) + id = 1, date = LocalDateTime.of(2020, 2, 1, 13, 37), + position = Position(0.1, 0.1), + driver = jonSmith, vehicle = tesla, + vehicleData = VehicleData(27.7778, Consumption.Electric(18.1)), ), LocationRecord( - 9_000, LocalDateTime.of(2020, 2, 5, 7, 59), - Position(48.137154, 11.576124), - janeDoe, harley, - VehicleData(20.0, Consumption.Combustion(7.9)) + id = 9_000, date = LocalDateTime.of(2020, 2, 5, 7, 59), + position = Position(48.137154, 11.576124), + driver = janeDoe, vehicle = harley, + vehicleData = VehicleData(20.0, Consumption.Combustion(7.9)), ) ), - ListSerializer(LocationRecord.serializer()) + serializer = ListSerializer(LocationRecord.serializer()) ) @Test @@ -83,17 +98,18 @@ class ExampleTest { polymorphic(Part::class, Oil::class, Oil.serializer()) } }.assertEncodeAndDecode( - """|101,f9682dcb-30f7-4e88-915e-60e3b2758da7,CAR,Tesla,Tire,FRONT,LEFT,245,35,21,0.25 - |102,f9682dcb-30f7-4e88-915e-60e3b2758da7,CAR,Tesla,Tire,FRONT,RIGHT,245,35,21,0.21 - |103,f9682dcb-30f7-4e88-915e-60e3b2758da7,CAR,Tesla,Tire,REAR,LEFT,265,35,21,0.35 - |104,f9682dcb-30f7-4e88-915e-60e3b2758da7,CAR,Tesla,Tire,REAR,RIGHT,265,35,21,0.32 - |201,5e1afd88-97a2-4373-a83c-44a49c552abd,CAR,Porsche,Oil,20,50,0.2 - |202,5e1afd88-97a2-4373-a83c-44a49c552abd,CAR,Porsche,Tire,FRONT,LEFT,265,35,20,0.2 - |203,5e1afd88-97a2-4373-a83c-44a49c552abd,CAR,Porsche,Tire,FRONT,RIGHT,265,35,20,0.2 - |204,5e1afd88-97a2-4373-a83c-44a49c552abd,CAR,Porsche,Tire,REAR,LEFT,265,35,20,0.2 - |205,5e1afd88-97a2-4373-a83c-44a49c552abd,CAR,Porsche,Tire,REAR,RIGHT,265,35,20,0.2 + expected = """ + |101,f9682dcb-30f7-4e88-915e-60e3b2758da7,CAR,Tesla,Tire,FRONT,LEFT,245,35,21,0.25 + |102,f9682dcb-30f7-4e88-915e-60e3b2758da7,CAR,Tesla,Tire,FRONT,RIGHT,245,35,21,0.21 + |103,f9682dcb-30f7-4e88-915e-60e3b2758da7,CAR,Tesla,Tire,REAR,LEFT,265,35,21,0.35 + |104,f9682dcb-30f7-4e88-915e-60e3b2758da7,CAR,Tesla,Tire,REAR,RIGHT,265,35,21,0.32 + |201,5e1afd88-97a2-4373-a83c-44a49c552abd,CAR,Porsche,Oil,20,50,0.2 + |202,5e1afd88-97a2-4373-a83c-44a49c552abd,CAR,Porsche,Tire,FRONT,LEFT,265,35,20,0.2 + |203,5e1afd88-97a2-4373-a83c-44a49c552abd,CAR,Porsche,Tire,FRONT,RIGHT,265,35,20,0.2 + |204,5e1afd88-97a2-4373-a83c-44a49c552abd,CAR,Porsche,Tire,REAR,LEFT,265,35,20,0.2 + |205,5e1afd88-97a2-4373-a83c-44a49c552abd,CAR,Porsche,Tire,REAR,RIGHT,265,35,20,0.2 """.trimMargin(), - listOf( + original = listOf( VehiclePartRecord(101, tesla, Tire(FRONT, LEFT, 245, 35, 21), 0.25), VehiclePartRecord(102, tesla, Tire(FRONT, RIGHT, 245, 35, 21), 0.21), VehiclePartRecord(103, tesla, Tire(REAR, LEFT, 265, 35, 21), 0.35), @@ -104,44 +120,152 @@ class ExampleTest { VehiclePartRecord(204, porsche, Tire(REAR, LEFT, 265, 35, 20), 0.2), VehiclePartRecord(205, porsche, Tire(REAR, RIGHT, 265, 35, 20), 0.2) ), - ListSerializer(VehiclePartRecord.serializer()) + serializer = ListSerializer(VehiclePartRecord.serializer()) ) @Test fun testVehicleFeaturesRecords() = Csv.assertEncodeAndDecode( - """|c038c27b-a3fd-4e35-b6ac-ab06d747e16c,MOTORBIKE,Harley,, - |c038c27b-a3fd-4e35-b6ac-ab06d747e16c,MOTORBIKE,Harley,0,0 - |f9682dcb-30f7-4e88-915e-60e3b2758da7,CAR,Tesla,5,ELECTRIC,AUTOMATIC,HEATED_SEATS,NAVIGATION_SYSTEM,XENON,2,ELECTRIC,0,XENON,1 + expected = """ + |c038c27b-a3fd-4e35-b6ac-ab06d747e16c,MOTORBIKE,Harley,, + |c038c27b-a3fd-4e35-b6ac-ab06d747e16c,MOTORBIKE,Harley,0,0 + |f9682dcb-30f7-4e88-915e-60e3b2758da7,CAR,Tesla,5,ELECTRIC,AUTOMATIC,HEATED_SEATS,NAVIGATION_SYSTEM,XENON,2,ELECTRIC,0,XENON,1 """.trimMargin(), - listOf( - VehicleFeaturesRecord(harley, null, null), - VehicleFeaturesRecord(harley, emptyList(), emptyMap()), + original = listOf( VehicleFeaturesRecord( - tesla, - listOf(ELECTRIC, AUTOMATIC, HEATED_SEATS, NAVIGATION_SYSTEM, XENON), - mapOf(ELECTRIC to 0, XENON to 1) + vehicle = harley, + features = null, + map = null, + ), + VehicleFeaturesRecord( + vehicle = harley, + features = emptyList(), + map = emptyMap(), + ), + VehicleFeaturesRecord( + vehicle = tesla, + features = listOf(ELECTRIC, AUTOMATIC, HEATED_SEATS, NAVIGATION_SYSTEM, XENON), + map = mapOf(ELECTRIC to 0, XENON to 1), ) ), - ListSerializer(VehicleFeaturesRecord.serializer()) + serializer = ListSerializer(VehicleFeaturesRecord.serializer()) ) @Test fun testRfc4180() = Csv.Rfc4180.assertEncodeAndDecode( - """|c038c27b-a3fd-4e35-b6ac-ab06d747e16c,MOTORBIKE,Harley,, - |c038c27b-a3fd-4e35-b6ac-ab06d747e16c,MOTORBIKE,Harley,0,0 - | - |f9682dcb-30f7-4e88-915e-60e3b2758da7,CAR,Tesla,5,ELECTRIC,AUTOMATIC,HEATED_SEATS,NAVIGATION_SYSTEM,XENON,2,ELECTRIC,0,XENON,1 + expected = """ + |c038c27b-a3fd-4e35-b6ac-ab06d747e16c,MOTORBIKE,Harley,, + |c038c27b-a3fd-4e35-b6ac-ab06d747e16c,MOTORBIKE,Harley,0,0 + | + |f9682dcb-30f7-4e88-915e-60e3b2758da7,CAR,Tesla,5,ELECTRIC,AUTOMATIC,HEATED_SEATS,NAVIGATION_SYSTEM,XENON,2,ELECTRIC,0,XENON,1 """.trimMargin().replace("\n", "\r\n"), - listOf( - VehicleFeaturesRecord(harley, null, null), - VehicleFeaturesRecord(harley, emptyList(), emptyMap()), + original = listOf( + VehicleFeaturesRecord( + vehicle = harley, + features = null, + map = null, + ), + VehicleFeaturesRecord( + vehicle = harley, + features = emptyList(), + map = emptyMap(), + ), null, VehicleFeaturesRecord( - tesla, - listOf(ELECTRIC, AUTOMATIC, HEATED_SEATS, NAVIGATION_SYSTEM, XENON), - mapOf(ELECTRIC to 0, XENON to 1) + vehicle = tesla, + features = listOf(ELECTRIC, AUTOMATIC, HEATED_SEATS, NAVIGATION_SYSTEM, XENON), + map = mapOf(ELECTRIC to 0, XENON to 1), ) ), - ListSerializer(VehicleFeaturesRecord.serializer().nullable) + serializer = ListSerializer(VehicleFeaturesRecord.serializer().nullable) ) + + @Test + fun testStreaming() = runTest { + val csv = Csv { + serializersModule = SerializersModule { + polymorphic(Part::class, Tire::class, Tire.serializer()) + polymorphic(Part::class, Oil::class, Oil.serializer()) + } + } + val testData = listOf( + VehiclePartRecord(101, tesla, Tire(FRONT, LEFT, 245, 35, 21), 0.25), + VehiclePartRecord(102, tesla, Tire(FRONT, RIGHT, 245, 35, 21), 0.21), + VehiclePartRecord(103, tesla, Tire(REAR, LEFT, 265, 35, 21), 0.35), + VehiclePartRecord(104, tesla, Tire(REAR, RIGHT, 265, 35, 21), 0.32), + VehiclePartRecord(201, porsche, Oil(20, 50), 0.2), + VehiclePartRecord(202, porsche, Tire(FRONT, LEFT, 265, 35, 20), 0.2), + VehiclePartRecord(203, porsche, Tire(FRONT, RIGHT, 265, 35, 20), 0.2), + VehiclePartRecord(204, porsche, Tire(REAR, LEFT, 265, 35, 20), 0.2), + VehiclePartRecord(205, porsche, Tire(REAR, RIGHT, 265, 35, 20), 0.2) + ) + + val input = PipedReader() + val sink = PipedWriter(input) + + val writer = csv.recordWriter(VehiclePartRecord.serializer(), sink) + val reader = csv.recordReader(VehiclePartRecord.serializer(), input) + + val readerTask = async(Dispatchers.IO) { + reader.asSequence().toList() + } + val writerTask = async(Dispatchers.IO) { + sink.use { output -> + testData.forEach { + writer.write(it) + output.flush() + } + } + } + + writerTask.await() + val result = readerTask.await() + + assertEquals(testData, result) + } + + @Test + fun testStreamingHeaders() = runTest { + val csv = Csv { + hasHeaderRecord = true + } + val testData = listOf( + Tire(FRONT, LEFT, 245, 35, 21), + Tire(FRONT, RIGHT, 245, 35, 21), + Tire(REAR, LEFT, 265, 35, 21), + Tire(REAR, RIGHT, 265, 35, 21), + Tire(FRONT, LEFT, 265, 35, 20), + Tire(FRONT, RIGHT, 265, 35, 20), + Tire(REAR, LEFT, 265, 35, 20), + Tire(REAR, RIGHT, 265, 35, 20) + ) + + val source = PipedReader() + val sink = PipedWriter(source) + + val readerTask = async(Dispatchers.IO) { + source.buffered().let { input -> + val reader = csv.recordReader(Tire.serializer(), input) + reader.asSequence().onEach { + println("Read $it") + }.toList() + } + } + + val writerTask = async(Dispatchers.IO) { + sink.buffered().use { output -> + val writer = csv.recordWriter(Tire.serializer(), output) + testData.forEach { + println("Writing $it") + writer.write(it) + output.flush() + delay(1.milliseconds) + } + } + } + + writerTask.await() + val result = readerTask.await() + + assertEquals(testData, result) + } }