1.0.38 (#42)

* 1.0.38

* ui api update

* ImageActor

* text input

README.md

@@ -76,18 +76,18 @@ Maven:
 <dependency>
     <groupId>com.simiacryptus</groupId>
     <artifactId>skyenet-webui</artifactId>
-    <version>1.0.37</version>
+    <version>1.0.38</version>
 </dependency>
 ```
 
 Gradle:
 
 ```groovy
-implementation group: 'com.simiacryptus', name: 'skyenet', version: '1.0.37'
+implementation group: 'com.simiacryptus', name: 'skyenet', version: '1.0.38'
 ```
 
 ```kotlin
-implementation("com.simiacryptus:skyenet:1.0.37")
+implementation("com.simiacryptus:skyenet:1.0.38")
 ```
 
 ### 🌟 To Use

core/build.gradle.kts

@@ -31,7 +31,7 @@ val logback_version = "1.4.11"
 
 dependencies {
 
-    implementation(group = "com.simiacryptus", name = "jo-penai", version = "1.0.34")
+    implementation(group = "com.simiacryptus", name = "jo-penai", version = "1.0.35")
 
     implementation(group = "org.slf4j", name = "slf4j-api", version = "2.0.9")
     implementation(group = "commons-io", name = "commons-io", version = "2.15.0")

core/src/main/kotlin/com/simiacryptus/skyenet/core/actors/ActorSystem.kt

@@ -1,6 +1,7 @@
 package com.simiacryptus.skyenet.core.actors
 
 import com.simiacryptus.skyenet.core.actors.record.CodingActorInterceptor
+import com.simiacryptus.skyenet.core.actors.record.ImageActorInterceptor
 import com.simiacryptus.skyenet.core.actors.record.ParsedActorInterceptor
 import com.simiacryptus.skyenet.core.actors.record.SimpleActorInterceptor
 import com.simiacryptus.skyenet.core.platform.DataStorage
@@ -13,17 +14,18 @@ import java.io.File
 open class ActorSystem<T:Enum<*>>(
     private val actors: Map<T, BaseActor<*>>,
     val dataStorage: DataStorage,
-    val userId: User?,
-    val sessionId: Session
+    val user: User?,
+    val session: Session
 ) {
-    private val sessionDir = dataStorage.getSessionDir(userId, sessionId)
+    private val sessionDir = dataStorage.getSessionDir(user, session)
     fun getActor(actor: T): BaseActor<*> {
         val wrapper = getWrapper(actor.name)
         return when (val baseActor = actors[actor]) {
             null -> throw RuntimeException("No actor for $actor")
             is SimpleActor -> SimpleActorInterceptor(baseActor, wrapper)
             is ParsedActor<*> -> ParsedActorInterceptor(baseActor, wrapper)
             is CodingActor -> CodingActorInterceptor(baseActor, wrapper)
+            is ImageActor -> ImageActorInterceptor(baseActor, wrapper)
             else -> throw RuntimeException("Unknown actor type: ${baseActor.javaClass}")
         }
     }

core/src/main/kotlin/com/simiacryptus/skyenet/core/actors/BaseActor.kt

@@ -1,5 +1,6 @@
 package com.simiacryptus.skyenet.core.actors
 
+import com.fasterxml.jackson.annotation.JsonIgnore
 import com.simiacryptus.jopenai.API
 import com.simiacryptus.jopenai.ClientUtil.toContentList
 import com.simiacryptus.jopenai.OpenAIClient
@@ -10,7 +11,7 @@ import com.simiacryptus.jopenai.models.OpenAITextModel
 abstract class BaseActor<T>(
     open val prompt: String,
     val name: String? = null,
-    val model: OpenAITextModel = ChatModels.GPT35Turbo,
+    val model: ChatModels = ChatModels.GPT35Turbo,
     val temperature: Double = 0.3,
 ) {
     abstract fun answer(vararg messages: com.simiacryptus.jopenai.ApiModel.ChatMessage, api: API): T

core/src/main/kotlin/com/simiacryptus/skyenet/core/actors/CodingActor.kt

@@ -2,6 +2,7 @@ package com.simiacryptus.skyenet.core.actors
 
 import com.fasterxml.jackson.annotation.JsonIgnore
 import com.simiacryptus.jopenai.API
+import com.simiacryptus.jopenai.ApiModel.*
 import com.simiacryptus.jopenai.ClientUtil.toContentList
 import com.simiacryptus.jopenai.OpenAIClient
 import com.simiacryptus.jopenai.describe.AbbrevWhitelistYamlDescriber
@@ -12,6 +13,7 @@ import com.simiacryptus.skyenet.core.Brain
 import com.simiacryptus.skyenet.core.Brain.Companion.indent
 import com.simiacryptus.skyenet.core.Heart
 import com.simiacryptus.skyenet.core.OutputInterceptor
+import javax.script.ScriptException
 import kotlin.reflect.KClass
 
 @Suppress("unused", "MemberVisibilityCanBePrivate")
@@ -24,7 +26,7 @@ open class CodingActor(
     ),
     name: String? = interpreterClass.simpleName,
     val details: String? = null,
-    model: OpenAITextModel = ChatModels.GPT35Turbo,
+    model: ChatModels = ChatModels.GPT35Turbo,
     val fallbackModel: OpenAITextModel = ChatModels.GPT4Turbo,
     temperature: Double = 0.1,
     val autoEvaluate: Boolean = false,
@@ -71,13 +73,13 @@ open class CodingActor(
         if (!autoEvaluate) answer(*chatMessages(*questions), api = api)
         else answerWithAutoEval(*chatMessages(*questions), api = api).first
 
-    override fun answer(vararg messages: com.simiacryptus.jopenai.ApiModel.ChatMessage, api: API): CodeResult =
+    override fun answer(vararg messages: ChatMessage, api: API): CodeResult =
         if (!autoEvaluate) CodeResultImpl(*messages, api = (api as OpenAIClient))
         else answerWithAutoEval(*messages, api = api).first
 
     open fun answerWithPrefix(
         codePrefix: String,
-        vararg messages: com.simiacryptus.jopenai.ApiModel.ChatMessage,
+        vararg messages: ChatMessage,
         api: API
     ): CodeResult =
         if (!autoEvaluate) CodeResultImpl(*injectCodePrefix(messages, codePrefix), api = (api as OpenAIClient))
@@ -90,7 +92,7 @@ open class CodingActor(
     ) = answerWithAutoEval(*injectCodePrefix(chatMessages(*messages), codePrefix), api = api)
 
     open fun answerWithAutoEval(
-        vararg messages: com.simiacryptus.jopenai.ApiModel.ChatMessage,
+        vararg messages: ChatMessage,
         api: API
     ): Pair<CodeResult, ExecutionResult> {
         var result = CodeResultImpl(*messages, api = (api as OpenAIClient))
@@ -117,15 +119,15 @@ open class CodingActor(
     }
 
     private fun injectCodePrefix(
-        messages: Array<out com.simiacryptus.jopenai.ApiModel.ChatMessage>,
+        messages: Array<out ChatMessage>,
         codePrefix: String
     ) = (messages.dropLast(1) + if (codePrefix.isBlank()) listOf() else listOf(
-        com.simiacryptus.jopenai.ApiModel.ChatMessage(com.simiacryptus.jopenai.ApiModel.Role.assistant, codePrefix.toContentList())
+        ChatMessage(Role.assistant, codePrefix.toContentList())
     ) + messages.last()).toTypedArray()
 
     private fun fix(
         api: OpenAIClient,
-        messages: Array<out com.simiacryptus.jopenai.ApiModel.ChatMessage>,
+        messages: Array<out ChatMessage>,
         result: CodeResultImpl,
         ex: Throwable
     ): CodeResultImpl {
@@ -147,22 +149,22 @@ open class CodingActor(
     )
 
     private inner class CodeResultImpl(
-        vararg messages: com.simiacryptus.jopenai.ApiModel.ChatMessage,
+        vararg messages: ChatMessage,
         codePrefix: String = "",
         api: OpenAIClient,
     ) : CodeResult {
-        private var _status = CodeResult.Status.Coding
+        var _status = CodeResult.Status.Coding
         override fun getStatus(): CodeResult.Status {
             return _status
         }
 
         private val impl by lazy {
             var codedInstruction = implement(
-                brain(api, model), *messages, codePrefix = codePrefix
+                this, brain(api, model), messages, codePrefix = codePrefix
             )
             if (_status != CodeResult.Status.Success && fallbackModel != model) {
                 codedInstruction = implement(
-                    brain(api, fallbackModel), *messages, codePrefix = codePrefix
+                    this, brain(api, fallbackModel), messages, codePrefix = codePrefix
                 )
             }
             if (_status != CodeResult.Status.Success) {
@@ -172,71 +174,72 @@ open class CodingActor(
             codedInstruction
         }
 
+        @JsonIgnore
+        override fun getCode(): String = impl
 
-        fun implement(
-            brain: Brain,
-            vararg messages: com.simiacryptus.jopenai.ApiModel.ChatMessage,
-            codePrefix: String = "",
-        ): String {
-            val response = brain.implement(*messages)
-            val codeBlocks = Brain.extractCodeBlocks(response)
-            for (codingAttempt in 0..fixRetries) {
-                var renderedResponse = getRenderedResponse(codeBlocks)
-                val codedInstruction = getCode(interpreter.getLanguage(), codeBlocks)
-                log.info("Response: \n\t${renderedResponse.replace("\n", "\n\t", false)}".trimMargin())
-                log.info("Code: \n\t${codedInstruction.replace("\n", "\n\t", false)}".trimMargin())
-                return validateAndFix(brain, codePrefix, codedInstruction, messages) ?: continue
-            }
-            return ""
-        }
+        override fun run() = execute(getCode())
+    }
 
-        fun validateAndFix(
-            brain: Brain,
-            codePrefix: String,
-            initialCode: String,
-            messages: Array<out com.simiacryptus.jopenai.ApiModel.ChatMessage>
-        ): String? {
-            var workingCode = initialCode
-            for (fixAttempt in 0..fixIterations) {
-                try {
-                    val validate = interpreter.validate((codePrefix + "\n" + workingCode).trim())
-                    if (validate != null) throw validate
-                    log.info("Validation succeeded")
-                    _status = CodeResult.Status.Success
-                    return workingCode
-                } catch (ex: Throwable) {
-                    log.info("Validation failed - ${ex.message}")
-                    _status = CodeResult.Status.Correcting
-                    val respondWithCode = brain.fixCommand(workingCode, ex, "", *messages)
-                    val response = getRenderedResponse(respondWithCode.second)
-                    workingCode = getCode(interpreter.getLanguage(), respondWithCode.second)
-                    log.info("Response: \n\t${response.replace("\n", "\n\t", false)}".trimMargin())
-                    log.info("Code: \n\t${workingCode.replace("\n", "\n\t", false)}".trimMargin())
-                }
-            }
-            return null
+    open fun implement(
+        self:CodeResult,
+        brain: Brain,
+        messages: Array<out ChatMessage>,
+        codePrefix: String
+    ): String {
+        val response = brain.implement(*messages)
+        val codeBlocks = Brain.extractCodeBlocks(response)
+        for (codingAttempt in 0..fixRetries) {
+            var renderedResponse = getRenderedResponse(codeBlocks)
+            val codedInstruction = getCode(interpreter.getLanguage(), codeBlocks)
+            log.info("Response: \n\t${renderedResponse.replace("\n", "\n\t", false)}".trimMargin())
+            log.info("Code: \n\t${codedInstruction.replace("\n", "\n\t", false)}".trimMargin())
+            return validateAndFix(self, codedInstruction, codePrefix, brain, messages) ?: continue
         }
+        return ""
+    }
 
-        @JsonIgnore
-        override fun getCode(): String {
-            return impl
+    open fun validateAndFix(
+        self : CodeResult,
+        initialCode: String,
+        codePrefix: String,
+        brain: Brain,
+        messages: Array<out ChatMessage>
+    ): String? {
+        var workingCode = initialCode
+        for (fixAttempt in 0..fixIterations) {
+            try {
+                val validate = interpreter.validate((codePrefix + "\n" + workingCode).trim())
+                if (validate != null) throw validate
+                log.info("Validation succeeded")
+                (self as CodeResultImpl)._status = CodeResult.Status.Success
+                return workingCode
+            } catch (ex: Throwable) {
+                log.info("Validation failed - ${ex.message}")
+                (self as CodeResultImpl)._status = CodeResult.Status.Correcting
+                val respondWithCode = brain.fixCommand(workingCode, ex, "", *messages)
+                val response = getRenderedResponse(respondWithCode.second)
+                workingCode = getCode(interpreter.getLanguage(), respondWithCode.second)
+                log.info("Response: \n\t${response.replace("\n", "\n\t", false)}".trimMargin())
+                log.info("Code: \n\t${workingCode.replace("\n", "\n\t", false)}".trimMargin())
+            }
         }
+        return null
+    }
 
-        override fun run(): ExecutionResult {
-            //language=HTML
-            log.info("Running ${getCode()}")
-            OutputInterceptor.clearGlobalOutput()
-            val result = try {
-                interpreter.run(getCode())
-            } catch (ex: javax.script.ScriptException) {
-                throw RuntimeException(errorMessage(getCode(), ex.lineNumber, ex.columnNumber, ex.message ?: ""), ex)
-            }
-            log.info("Result: $result")
-            //language=HTML
-            val executionResult = ExecutionResult(result.toString(), OutputInterceptor.getGlobalOutput())
-            OutputInterceptor.clearGlobalOutput()
-            return executionResult
+    open fun execute(code: String): ExecutionResult {
+        //language=HTML
+        log.info("Running $code")
+        OutputInterceptor.clearGlobalOutput()
+        val result = try {
+            interpreter.run(code)
+        } catch (ex: ScriptException) {
+            throw RuntimeException(errorMessage(code, ex.lineNumber, ex.columnNumber, ex.message ?: ""), ex)
         }
+        log.info("Result: $result")
+        //language=HTML
+        val executionResult = ExecutionResult(result.toString(), OutputInterceptor.getGlobalOutput())
+        OutputInterceptor.clearGlobalOutput()
+        return executionResult
     }
 
     companion object {

core/src/main/kotlin/com/simiacryptus/skyenet/core/actors/ImageActor.kt

@@ -0,0 +1,52 @@
+package com.simiacryptus.skyenet.core.actors
+
+import com.simiacryptus.jopenai.API
+import com.simiacryptus.jopenai.ApiModel
+import com.simiacryptus.jopenai.ApiModel.*
+import com.simiacryptus.jopenai.OpenAIClient
+import com.simiacryptus.jopenai.models.ChatModels
+import com.simiacryptus.jopenai.models.ImageModels
+import com.simiacryptus.jopenai.models.OpenAITextModel
+import com.simiacryptus.jopenai.proxy.ChatProxy
+import java.awt.image.BufferedImage
+import java.util.function.Function
+
+open class ImageActor(
+    prompt: String = "Transform the user request into an image generation prompt that the user will like",
+    val action: String? = null,
+    textModel: ChatModels = ChatModels.GPT35Turbo,
+    val imageModel: ImageModels = ImageModels.DallE2,
+    temperature: Double = 0.3,
+    val width: Int = 1024,
+    val height: Int = 1024,
+) : BaseActor<ImageResponse>(
+    prompt = prompt,
+    name = action,
+    model = textModel,
+    temperature = temperature,
+) {
+    private inner class ImageResponseImpl(vararg messages: ChatMessage, val api: API) : ImageResponse {
+
+        private val _text: String by lazy { response(*messages, api = api).choices.first().message?.content ?: throw RuntimeException("No response") }
+        override fun getText(): String = _text
+        override fun getImage(): BufferedImage {
+            val url = (api as OpenAIClient).createImage(
+                ImageGenerationRequest(
+                    prompt = getText(),
+                    model = imageModel.modelName,
+                    size = "${width}x$height"
+                )
+            ).data.first().url
+            return javax.imageio.ImageIO.read(java.net.URL(url))
+        }
+    }
+
+    override fun answer(vararg messages: ChatMessage, api: API): ImageResponse {
+        return ImageResponseImpl(*messages, api = api)
+    }
+}
+
+interface ImageResponse {
+    fun getText(): String
+    fun getImage(): BufferedImage
+}

core/src/main/kotlin/com/simiacryptus/skyenet/core/actors/ParsedActor.kt

@@ -11,7 +11,7 @@ open class ParsedActor<T:Any>(
     val parserClass: Class<out Function<String, T>>,
     prompt: String,
     val action: String? = null,
-    model: OpenAITextModel = ChatModels.GPT35Turbo,
+    model: ChatModels = ChatModels.GPT35Turbo,
     temperature: Double = 0.3,
 ) : BaseActor<ParsedResponse<T>>(
     prompt = prompt,

core/src/main/kotlin/com/simiacryptus/skyenet/core/actors/SimpleActor.kt

@@ -7,7 +7,7 @@ import com.simiacryptus.jopenai.models.OpenAITextModel
 open class SimpleActor(
     prompt: String,
     name: String? = null,
-    model: OpenAITextModel = ChatModels.GPT35Turbo,
+    model: ChatModels = ChatModels.GPT35Turbo,
     temperature: Double = 0.3,
 ) : BaseActor<String>(
     prompt = prompt,