Serialize BufferedImages as base64

core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/io/BytesUtils.kt

@@ -0,0 +1,5 @@
+package org.jetbrains.kotlinx.dataframe.impl.io
+
+import java.util.Base64
+
+internal fun ByteArray.toBase64(): String = Base64.getEncoder().encodeToString(this)

core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/io/compression.kt

@@ -0,0 +1,11 @@
+package org.jetbrains.kotlinx.dataframe.impl.io
+
+import java.io.ByteArrayOutputStream
+import java.util.zip.GZIPOutputStream
+
+internal fun ByteArray.encodeGzip(): ByteArray {
+    val bos = ByteArrayOutputStream()
+    GZIPOutputStream(bos).use { it.write(this) }
+
+    return bos.toByteArray()
+}

core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/io/image.kt

@@ -0,0 +1,60 @@
+package org.jetbrains.kotlinx.dataframe.impl.io
+
+import java.awt.Graphics2D
+import java.awt.RenderingHints
+import java.awt.image.BufferedImage
+import java.awt.image.ImageObserver
+import java.io.ByteArrayOutputStream
+import javax.imageio.ImageIO
+import kotlin.math.max
+import kotlin.math.min
+
+internal fun BufferedImage.resizeKeepingAspectRatio(
+    maxSize: Int,
+    resultImageType: Int = BufferedImage.TYPE_INT_ARGB,
+    interpolation: Any = RenderingHints.VALUE_INTERPOLATION_NEAREST_NEIGHBOR,
+    renderingQuality: Any = RenderingHints.VALUE_RENDER_QUALITY,
+    antialiasing: Any = RenderingHints.VALUE_ANTIALIAS_ON,
+    observer: ImageObserver? = null
+): BufferedImage {
+    val aspectRatio = width.toDouble() / height.toDouble()
+    val size = min(maxSize, max(width, height))
+
+    val (nWidth, nHeight) = if (width > height) {
+        Pair(size, (size / aspectRatio).toInt())
+    } else {
+        Pair((size * aspectRatio).toInt(), size)
+    }
+
+    return resize(nWidth, nHeight, resultImageType, interpolation, renderingQuality, antialiasing, observer)
+}
+
+internal fun BufferedImage.resize(
+    width: Int,
+    height: Int,
+    resultImageType: Int = BufferedImage.TYPE_INT_ARGB,
+    interpolation: Any = RenderingHints.VALUE_INTERPOLATION_NEAREST_NEIGHBOR,
+    renderingQuality: Any = RenderingHints.VALUE_RENDER_QUALITY,
+    antialiasing: Any = RenderingHints.VALUE_ANTIALIAS_ON,
+    observer: ImageObserver? = null
+): BufferedImage {
+    val resized = BufferedImage(width, height, resultImageType)
+    val g: Graphics2D = resized.createGraphics()
+
+    g.setRenderingHint(RenderingHints.KEY_INTERPOLATION, interpolation)
+    g.setRenderingHint(RenderingHints.KEY_RENDERING, renderingQuality)
+    g.setRenderingHint(RenderingHints.KEY_ANTIALIASING, antialiasing)
+
+    g.drawImage(this, 0, 0, width, height, observer)
+    g.dispose()
+
+    return resized
+}
+
+internal const val DEFAULT_IMG_FORMAT = "png"
+
+internal fun BufferedImage.toByteArray(format: String = DEFAULT_IMG_FORMAT): ByteArray =
+    ByteArrayOutputStream().use { bos ->
+        ImageIO.write(this, format, bos)
+        bos.toByteArray()
+    }

core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/io/writeJson.kt

@@ -23,11 +23,14 @@ import org.jetbrains.kotlinx.dataframe.impl.io.SerializationKeys.METADATA
 import org.jetbrains.kotlinx.dataframe.impl.io.SerializationKeys.NCOL
 import org.jetbrains.kotlinx.dataframe.impl.io.SerializationKeys.NROW
 import org.jetbrains.kotlinx.dataframe.impl.io.SerializationKeys.VERSION
+import org.jetbrains.kotlinx.dataframe.io.ImageEncodingOptions
 import org.jetbrains.kotlinx.dataframe.io.arrayColumnName
 import org.jetbrains.kotlinx.dataframe.io.valueColumnName
 import org.jetbrains.kotlinx.dataframe.ncol
 import org.jetbrains.kotlinx.dataframe.nrow
 import org.jetbrains.kotlinx.dataframe.typeClass
+import java.awt.image.BufferedImage
+import java.io.IOException
 
 internal fun KlaxonJson.encodeRow(frame: ColumnsContainer<*>, index: Int): JsonObject? {
     val values = frame.columns().map { col ->
@@ -57,18 +60,19 @@ internal const val SERIALIZATION_VERSION = "2.0.0"
 internal fun KlaxonJson.encodeRowWithMetadata(
     frame: ColumnsContainer<*>,
     index: Int,
-    rowLimit: Int? = null
+    rowLimit: Int? = null,
+    imageEncodingOptions: ImageEncodingOptions = ImageEncodingOptions()
 ): JsonObject? {
     val values = frame.columns().map { col ->
         when (col) {
             is ColumnGroup<*> -> obj(
-                DATA to encodeRowWithMetadata(col, index, rowLimit),
+                DATA to encodeRowWithMetadata(col, index, rowLimit, imageEncodingOptions),
                 METADATA to obj(KIND to ColumnKind.Group.toString())
             )
 
             is FrameColumn<*> -> {
-                val data = if (rowLimit == null) encodeFrameWithMetadata(col[index])
-                else encodeFrameWithMetadata(col[index].take(rowLimit), rowLimit)
+                val data = if (rowLimit == null) encodeFrameWithMetadata(col[index], null, imageEncodingOptions)
+                else encodeFrameWithMetadata(col[index].take(rowLimit), rowLimit, imageEncodingOptions)
                 obj(
                     DATA to data,
                     METADATA to obj(
@@ -79,7 +83,7 @@ internal fun KlaxonJson.encodeRowWithMetadata(
                 )
             }
 
-            else -> encodeValue(col, index)
+            else -> encodeValue(col, index, imageEncodingOptions)
         }.let { col.name to it }
     }
     if (values.isEmpty()) return null
@@ -89,7 +93,11 @@ internal fun KlaxonJson.encodeRowWithMetadata(
 private val valueTypes =
     setOf(Boolean::class, Double::class, Int::class, Float::class, Long::class, Short::class, Byte::class)
 
-internal fun KlaxonJson.encodeValue(col: AnyCol, index: Int): Any? = when {
+internal fun KlaxonJson.encodeValue(
+    col: AnyCol,
+    index: Int,
+    imageEncodingOptions: ImageEncodingOptions = ImageEncodingOptions(encodeAsBase64 = false)
+): Any? = when {
     col.isList() -> col[index]?.let { list ->
         val values = (list as List<*>).map {
             when (it) {
@@ -108,10 +116,45 @@ internal fun KlaxonJson.encodeValue(col: AnyCol, index: Int): Any? = when {
         } else v
     }
 
+    col.typeClass == BufferedImage::class -> col[index]?.let { image ->
+        encodeBufferedImage(image as BufferedImage, imageEncodingOptions)
+    } ?: ""
+
     else -> col[index]?.toString()
 }
 
-internal fun KlaxonJson.encodeFrameWithMetadata(frame: AnyFrame, rowLimit: Int? = null): JsonArray<*> {
+private fun encodeBufferedImage(
+    image: BufferedImage,
+    imageEncodingOptions: ImageEncodingOptions = ImageEncodingOptions()
+): String? {
+    if (!imageEncodingOptions.encodeAsBase64) {
+        return image.toString()
+    }
+
+    return try {
+        val preparedImage = if (imageEncodingOptions.isLimitSizeOn) {
+            image.resizeKeepingAspectRatio(imageEncodingOptions.imageSizeLimit)
+        } else {
+            image
+        }
+
+        val bytes = if (imageEncodingOptions.isGzipOn) {
+            preparedImage.toByteArray().encodeGzip()
+        } else {
+            preparedImage.toByteArray()
+        }
+
+        bytes.toBase64()
+    } catch (e: IOException) {
+        null
+    }
+}
+
+internal fun KlaxonJson.encodeFrameWithMetadata(
+    frame: AnyFrame,
+    rowLimit: Int? = null,
+    imageEncodingOptions: ImageEncodingOptions = ImageEncodingOptions()
+): JsonArray<*> {
     val valueColumn = frame.extractValueColumn()
     val arrayColumn = frame.extractArrayColumn()
 
@@ -122,9 +165,13 @@ internal fun KlaxonJson.encodeFrameWithMetadata(frame: AnyFrame, rowLimit: Int?
             ?.get(rowIndex)
             ?: arrayColumn?.get(rowIndex)
                 ?.let {
-                    if (arraysAreFrames) encodeFrameWithMetadata(it as AnyFrame, rowLimit) else null
+                    if (arraysAreFrames) encodeFrameWithMetadata(
+                        it as AnyFrame,
+                        rowLimit,
+                        imageEncodingOptions
+                    ) else null
                 }
-            ?: encodeRowWithMetadata(frame, rowIndex, rowLimit)
+            ?: encodeRowWithMetadata(frame, rowIndex, rowLimit, imageEncodingOptions)
     }
 
     return array(data)
@@ -206,6 +253,7 @@ internal fun KlaxonJson.encodeDataFrameWithMetadata(
     frame: AnyFrame,
     rowLimit: Int,
     nestedRowLimit: Int? = null,
+    imageEncodingOptions: ImageEncodingOptions = ImageEncodingOptions()
 ): JsonObject {
     return obj(
         VERSION to SERIALIZATION_VERSION,
@@ -216,7 +264,8 @@ internal fun KlaxonJson.encodeDataFrameWithMetadata(
         ),
         KOTLIN_DATAFRAME to encodeFrameWithMetadata(
             frame.take(rowLimit),
-            rowLimit = nestedRowLimit
+            rowLimit = nestedRowLimit,
+            imageEncodingOptions
         ),
     )
 }

core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/json.kt

@@ -276,20 +276,48 @@ public fun AnyFrame.toJson(prettyPrint: Boolean = false, canonical: Boolean = fa
  * Applied for each frame column recursively
  * @param prettyPrint Specifies whether the output JSON should be formatted with indentation and line breaks.
  * @param canonical Specifies whether the output JSON should be in a canonical form.
+ * @param imageEncodingOptions The options for encoding images in the DataFrame. Defaults to encode images as Base64.
  *
  * @return The DataFrame converted to a JSON string with metadata.
  */
 public fun AnyFrame.toJsonWithMetadata(
     rowLimit: Int,
     nestedRowLimit: Int? = null,
     prettyPrint: Boolean = false,
-    canonical: Boolean = false
+    canonical: Boolean = false,
+    imageEncodingOptions: ImageEncodingOptions = ImageEncodingOptions(encodeAsBase64 = true)
 ): String {
     return json {
-        encodeDataFrameWithMetadata(this@toJsonWithMetadata, rowLimit, nestedRowLimit)
+        encodeDataFrameWithMetadata(this@toJsonWithMetadata, rowLimit, nestedRowLimit, imageEncodingOptions)
     }.toJsonString(prettyPrint, canonical)
 }
 
+internal const val DEFAULT_IMG_SIZE = 600
+
+/**
+ * Class representing the options for encoding images.
+ *
+ * @property encodeAsBase64 Specifies whether the images should be encoded as Base64. Defaults to false.
+ * @property imageSizeLimit The maximum size to which images should be resized. Defaults to the value of DEFAULT_IMG_SIZE.
+ * @property options Bitwise-OR of the [GZIP_ON] and [LIMIT_SIZE_ON] constants. Defaults to [GZIP_ON] or [LIMIT_SIZE_ON].
+ */
+public class ImageEncodingOptions(
+    public val encodeAsBase64: Boolean = false,
+    public val imageSizeLimit: Int = DEFAULT_IMG_SIZE,
+    private val options: Int = GZIP_ON or LIMIT_SIZE_ON
+) {
+    public val isGzipOn: Boolean
+        get() = options and GZIP_ON == GZIP_ON
+
+    public val isLimitSizeOn: Boolean
+        get() = options and LIMIT_SIZE_ON == LIMIT_SIZE_ON
+
+    public companion object {
+        public const val GZIP_ON: Int = 1 // 2^0
+        public const val LIMIT_SIZE_ON: Int = 2 // 2^1
+    }
+}
+
 public fun AnyRow.toJson(prettyPrint: Boolean = false, canonical: Boolean = false): String {
     return json {
         encodeRow(df(), index())

core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/jupyter/Integration.kt

@@ -41,12 +41,16 @@ import org.jetbrains.kotlinx.dataframe.dataTypes.IMG
 import org.jetbrains.kotlinx.dataframe.impl.codeGen.CodeGenerationReadResult
 import org.jetbrains.kotlinx.dataframe.impl.codeGen.urlCodeGenReader
 import org.jetbrains.kotlinx.dataframe.impl.createStarProjectedType
+import org.jetbrains.kotlinx.dataframe.impl.io.resizeKeepingAspectRatio
+import org.jetbrains.kotlinx.dataframe.impl.io.toBase64
+import org.jetbrains.kotlinx.dataframe.impl.io.toByteArray
 import org.jetbrains.kotlinx.dataframe.impl.renderType
 import org.jetbrains.kotlinx.dataframe.io.DataFrameHtmlData
 import org.jetbrains.kotlinx.dataframe.io.SupportedCodeGenerationFormat
 import org.jetbrains.kotlinx.dataframe.io.supportedFormats
 import org.jetbrains.kotlinx.jupyter.api.*
 import org.jetbrains.kotlinx.jupyter.api.libraries.*
+import java.awt.image.BufferedImage
 import kotlin.reflect.KClass
 import kotlin.reflect.KProperty
 import kotlin.reflect.KType
@@ -55,6 +59,8 @@ import kotlin.reflect.full.isSubtypeOf
 /** Users will get an error if their Kotlin Jupyter kernel is older than this version. */
 private const val MIN_KERNEL_VERSION = "0.11.0.198"
 
+private const val DEFAULT_HTML_IMG_SIZE = 100
+
 internal val newDataSchemas = mutableListOf<KClass<*>>()
 
 internal class Integration(
@@ -203,6 +209,19 @@ internal class Integration(
             }
         }
 
+        notebook.renderersProcessor.registerWithoutOptimizing(
+            createRenderer<BufferedImage> {
+                val src = buildString {
+                    append("""data:image/$DEFAULT_HTML_IMG_SIZE;base64,""")
+                    append(
+                        it.resizeKeepingAspectRatio(DEFAULT_HTML_IMG_SIZE).toByteArray().toBase64()
+                    )
+                }
+                HTML("""<img src="$src"/>""")
+            },
+            ProcessingPriority.HIGHER
+        )
+
         with(JupyterHtmlRenderer(config.display, this)) {
             render<DisableRowsLimitWrapper>(
                 { "DataRow: index = ${it.value.rowsCount()}, columnsCount = ${it.value.columnsCount()}" },