From d68f0ffd476ee2d73a4af71e8effe6e528df31b8 Mon Sep 17 00:00:00 2001 From: Andrew Charneski Date: Thu, 23 Nov 2023 03:02:08 -0500 Subject: [PATCH] 1.0.38 (#42) * 1.0.38 * ui api update * ImageActor * text input --- README.md | 6 +- core/build.gradle.kts | 2 +- .../skyenet/core/actors/ActorSystem.kt | 8 +- .../skyenet/core/actors/BaseActor.kt | 3 +- .../skyenet/core/actors/CodingActor.kt | 141 +++++++++--------- .../skyenet/core/actors/ImageActor.kt | 52 +++++++ .../skyenet/core/actors/ParsedActor.kt | 2 +- .../skyenet/core/actors/SimpleActor.kt | 2 +- .../actors/record/CodingActorInterceptor.kt | 87 +++++++---- .../actors/record/ImageActorInterceptor.kt | 43 ++++++ .../actors/record/ParsedActorInterceptor.kt | 16 +- .../core/actors/test/ImageActorTestBase.kt | 14 ++ .../skyenet/core/util/FunctionWrapper.kt | 17 +++ gradle.properties | 2 +- webui/build.gradle.kts | 2 +- .../webui/application/ApplicationInterface.kt | 8 +- .../webui/application/ApplicationServer.kt | 3 +- .../application/ApplicationSocketManager.kt | 7 +- .../skyenet/webui/chat/ChatSocketManager.kt | 3 +- .../skyenet/webui/session/SessionMessage.kt | 81 +++++++++- .../webui/session/SocketManagerBase.kt | 35 +++-- .../skyenet/webui/test/CodingActorTestApp.kt | 60 ++++---- .../skyenet/webui/test/ImageActorTestApp.kt | 52 +++++++ .../skyenet/webui/test/ParsedActorTestApp.kt | 30 ++-- .../skyenet/webui/test/SimpleActorTestApp.kt | 18 ++- ...ngVisualizer.kt => TensorflowProjector.kt} | 2 +- webui/src/main/resources/application/chat.css | 19 ++- .../skyenet/webui/ActorTestAppServer.kt | 3 + 28 files changed, 518 insertions(+), 200 deletions(-) create mode 100644 core/src/main/kotlin/com/simiacryptus/skyenet/core/actors/ImageActor.kt create mode 100644 core/src/main/kotlin/com/simiacryptus/skyenet/core/actors/record/ImageActorInterceptor.kt create mode 100644 core/src/main/kotlin/com/simiacryptus/skyenet/core/actors/test/ImageActorTestBase.kt create mode 100644 webui/src/main/kotlin/com/simiacryptus/skyenet/webui/test/ImageActorTestApp.kt rename webui/src/main/kotlin/com/simiacryptus/skyenet/webui/util/{EmbeddingVisualizer.kt => TensorflowProjector.kt} (99%) diff --git a/README.md b/README.md index 7858c7ee..639b6118 100644 --- a/README.md +++ b/README.md @@ -76,18 +76,18 @@ Maven: com.simiacryptus skyenet-webui - 1.0.37 + 1.0.38 ``` Gradle: ```groovy -implementation group: 'com.simiacryptus', name: 'skyenet', version: '1.0.37' +implementation group: 'com.simiacryptus', name: 'skyenet', version: '1.0.38' ``` ```kotlin -implementation("com.simiacryptus:skyenet:1.0.37") +implementation("com.simiacryptus:skyenet:1.0.38") ``` ### 🌟 To Use diff --git a/core/build.gradle.kts b/core/build.gradle.kts index 97fb7303..dac2105a 100644 --- a/core/build.gradle.kts +++ b/core/build.gradle.kts @@ -31,7 +31,7 @@ val logback_version = "1.4.11" dependencies { - implementation(group = "com.simiacryptus", name = "jo-penai", version = "1.0.34") + implementation(group = "com.simiacryptus", name = "jo-penai", version = "1.0.35") implementation(group = "org.slf4j", name = "slf4j-api", version = "2.0.9") implementation(group = "commons-io", name = "commons-io", version = "2.15.0") diff --git a/core/src/main/kotlin/com/simiacryptus/skyenet/core/actors/ActorSystem.kt b/core/src/main/kotlin/com/simiacryptus/skyenet/core/actors/ActorSystem.kt index 6bd1a647..ab26dd5f 100644 --- a/core/src/main/kotlin/com/simiacryptus/skyenet/core/actors/ActorSystem.kt +++ b/core/src/main/kotlin/com/simiacryptus/skyenet/core/actors/ActorSystem.kt @@ -1,6 +1,7 @@ package com.simiacryptus.skyenet.core.actors import com.simiacryptus.skyenet.core.actors.record.CodingActorInterceptor +import com.simiacryptus.skyenet.core.actors.record.ImageActorInterceptor import com.simiacryptus.skyenet.core.actors.record.ParsedActorInterceptor import com.simiacryptus.skyenet.core.actors.record.SimpleActorInterceptor import com.simiacryptus.skyenet.core.platform.DataStorage @@ -13,10 +14,10 @@ import java.io.File open class ActorSystem>( private val actors: Map>, val dataStorage: DataStorage, - val userId: User?, - val sessionId: Session + val user: User?, + val session: Session ) { - private val sessionDir = dataStorage.getSessionDir(userId, sessionId) + private val sessionDir = dataStorage.getSessionDir(user, session) fun getActor(actor: T): BaseActor<*> { val wrapper = getWrapper(actor.name) return when (val baseActor = actors[actor]) { @@ -24,6 +25,7 @@ open class ActorSystem>( is SimpleActor -> SimpleActorInterceptor(baseActor, wrapper) is ParsedActor<*> -> ParsedActorInterceptor(baseActor, wrapper) is CodingActor -> CodingActorInterceptor(baseActor, wrapper) + is ImageActor -> ImageActorInterceptor(baseActor, wrapper) else -> throw RuntimeException("Unknown actor type: ${baseActor.javaClass}") } } diff --git a/core/src/main/kotlin/com/simiacryptus/skyenet/core/actors/BaseActor.kt b/core/src/main/kotlin/com/simiacryptus/skyenet/core/actors/BaseActor.kt index c3785710..8a38d49e 100644 --- a/core/src/main/kotlin/com/simiacryptus/skyenet/core/actors/BaseActor.kt +++ b/core/src/main/kotlin/com/simiacryptus/skyenet/core/actors/BaseActor.kt @@ -1,5 +1,6 @@ package com.simiacryptus.skyenet.core.actors +import com.fasterxml.jackson.annotation.JsonIgnore import com.simiacryptus.jopenai.API import com.simiacryptus.jopenai.ClientUtil.toContentList import com.simiacryptus.jopenai.OpenAIClient @@ -10,7 +11,7 @@ import com.simiacryptus.jopenai.models.OpenAITextModel abstract class BaseActor( open val prompt: String, val name: String? = null, - val model: OpenAITextModel = ChatModels.GPT35Turbo, + val model: ChatModels = ChatModels.GPT35Turbo, val temperature: Double = 0.3, ) { abstract fun answer(vararg messages: com.simiacryptus.jopenai.ApiModel.ChatMessage, api: API): T diff --git a/core/src/main/kotlin/com/simiacryptus/skyenet/core/actors/CodingActor.kt b/core/src/main/kotlin/com/simiacryptus/skyenet/core/actors/CodingActor.kt index 943cbff0..d647da9a 100644 --- a/core/src/main/kotlin/com/simiacryptus/skyenet/core/actors/CodingActor.kt +++ b/core/src/main/kotlin/com/simiacryptus/skyenet/core/actors/CodingActor.kt @@ -2,6 +2,7 @@ package com.simiacryptus.skyenet.core.actors import com.fasterxml.jackson.annotation.JsonIgnore import com.simiacryptus.jopenai.API +import com.simiacryptus.jopenai.ApiModel.* import com.simiacryptus.jopenai.ClientUtil.toContentList import com.simiacryptus.jopenai.OpenAIClient import com.simiacryptus.jopenai.describe.AbbrevWhitelistYamlDescriber @@ -12,6 +13,7 @@ import com.simiacryptus.skyenet.core.Brain import com.simiacryptus.skyenet.core.Brain.Companion.indent import com.simiacryptus.skyenet.core.Heart import com.simiacryptus.skyenet.core.OutputInterceptor +import javax.script.ScriptException import kotlin.reflect.KClass @Suppress("unused", "MemberVisibilityCanBePrivate") @@ -24,7 +26,7 @@ open class CodingActor( ), name: String? = interpreterClass.simpleName, val details: String? = null, - model: OpenAITextModel = ChatModels.GPT35Turbo, + model: ChatModels = ChatModels.GPT35Turbo, val fallbackModel: OpenAITextModel = ChatModels.GPT4Turbo, temperature: Double = 0.1, val autoEvaluate: Boolean = false, @@ -71,13 +73,13 @@ open class CodingActor( if (!autoEvaluate) answer(*chatMessages(*questions), api = api) else answerWithAutoEval(*chatMessages(*questions), api = api).first - override fun answer(vararg messages: com.simiacryptus.jopenai.ApiModel.ChatMessage, api: API): CodeResult = + override fun answer(vararg messages: ChatMessage, api: API): CodeResult = if (!autoEvaluate) CodeResultImpl(*messages, api = (api as OpenAIClient)) else answerWithAutoEval(*messages, api = api).first open fun answerWithPrefix( codePrefix: String, - vararg messages: com.simiacryptus.jopenai.ApiModel.ChatMessage, + vararg messages: ChatMessage, api: API ): CodeResult = if (!autoEvaluate) CodeResultImpl(*injectCodePrefix(messages, codePrefix), api = (api as OpenAIClient)) @@ -90,7 +92,7 @@ open class CodingActor( ) = answerWithAutoEval(*injectCodePrefix(chatMessages(*messages), codePrefix), api = api) open fun answerWithAutoEval( - vararg messages: com.simiacryptus.jopenai.ApiModel.ChatMessage, + vararg messages: ChatMessage, api: API ): Pair { var result = CodeResultImpl(*messages, api = (api as OpenAIClient)) @@ -117,15 +119,15 @@ open class CodingActor( } private fun injectCodePrefix( - messages: Array, + messages: Array, codePrefix: String ) = (messages.dropLast(1) + if (codePrefix.isBlank()) listOf() else listOf( - com.simiacryptus.jopenai.ApiModel.ChatMessage(com.simiacryptus.jopenai.ApiModel.Role.assistant, codePrefix.toContentList()) + ChatMessage(Role.assistant, codePrefix.toContentList()) ) + messages.last()).toTypedArray() private fun fix( api: OpenAIClient, - messages: Array, + messages: Array, result: CodeResultImpl, ex: Throwable ): CodeResultImpl { @@ -147,22 +149,22 @@ open class CodingActor( ) private inner class CodeResultImpl( - vararg messages: com.simiacryptus.jopenai.ApiModel.ChatMessage, + vararg messages: ChatMessage, codePrefix: String = "", api: OpenAIClient, ) : CodeResult { - private var _status = CodeResult.Status.Coding + var _status = CodeResult.Status.Coding override fun getStatus(): CodeResult.Status { return _status } private val impl by lazy { var codedInstruction = implement( - brain(api, model), *messages, codePrefix = codePrefix + this, brain(api, model), messages, codePrefix = codePrefix ) if (_status != CodeResult.Status.Success && fallbackModel != model) { codedInstruction = implement( - brain(api, fallbackModel), *messages, codePrefix = codePrefix + this, brain(api, fallbackModel), messages, codePrefix = codePrefix ) } if (_status != CodeResult.Status.Success) { @@ -172,71 +174,72 @@ open class CodingActor( codedInstruction } + @JsonIgnore + override fun getCode(): String = impl - fun implement( - brain: Brain, - vararg messages: com.simiacryptus.jopenai.ApiModel.ChatMessage, - codePrefix: String = "", - ): String { - val response = brain.implement(*messages) - val codeBlocks = Brain.extractCodeBlocks(response) - for (codingAttempt in 0..fixRetries) { - var renderedResponse = getRenderedResponse(codeBlocks) - val codedInstruction = getCode(interpreter.getLanguage(), codeBlocks) - log.info("Response: \n\t${renderedResponse.replace("\n", "\n\t", false)}".trimMargin()) - log.info("Code: \n\t${codedInstruction.replace("\n", "\n\t", false)}".trimMargin()) - return validateAndFix(brain, codePrefix, codedInstruction, messages) ?: continue - } - return "" - } + override fun run() = execute(getCode()) + } - fun validateAndFix( - brain: Brain, - codePrefix: String, - initialCode: String, - messages: Array - ): String? { - var workingCode = initialCode - for (fixAttempt in 0..fixIterations) { - try { - val validate = interpreter.validate((codePrefix + "\n" + workingCode).trim()) - if (validate != null) throw validate - log.info("Validation succeeded") - _status = CodeResult.Status.Success - return workingCode - } catch (ex: Throwable) { - log.info("Validation failed - ${ex.message}") - _status = CodeResult.Status.Correcting - val respondWithCode = brain.fixCommand(workingCode, ex, "", *messages) - val response = getRenderedResponse(respondWithCode.second) - workingCode = getCode(interpreter.getLanguage(), respondWithCode.second) - log.info("Response: \n\t${response.replace("\n", "\n\t", false)}".trimMargin()) - log.info("Code: \n\t${workingCode.replace("\n", "\n\t", false)}".trimMargin()) - } - } - return null + open fun implement( + self:CodeResult, + brain: Brain, + messages: Array, + codePrefix: String + ): String { + val response = brain.implement(*messages) + val codeBlocks = Brain.extractCodeBlocks(response) + for (codingAttempt in 0..fixRetries) { + var renderedResponse = getRenderedResponse(codeBlocks) + val codedInstruction = getCode(interpreter.getLanguage(), codeBlocks) + log.info("Response: \n\t${renderedResponse.replace("\n", "\n\t", false)}".trimMargin()) + log.info("Code: \n\t${codedInstruction.replace("\n", "\n\t", false)}".trimMargin()) + return validateAndFix(self, codedInstruction, codePrefix, brain, messages) ?: continue } + return "" + } - @JsonIgnore - override fun getCode(): String { - return impl + open fun validateAndFix( + self : CodeResult, + initialCode: String, + codePrefix: String, + brain: Brain, + messages: Array + ): String? { + var workingCode = initialCode + for (fixAttempt in 0..fixIterations) { + try { + val validate = interpreter.validate((codePrefix + "\n" + workingCode).trim()) + if (validate != null) throw validate + log.info("Validation succeeded") + (self as CodeResultImpl)._status = CodeResult.Status.Success + return workingCode + } catch (ex: Throwable) { + log.info("Validation failed - ${ex.message}") + (self as CodeResultImpl)._status = CodeResult.Status.Correcting + val respondWithCode = brain.fixCommand(workingCode, ex, "", *messages) + val response = getRenderedResponse(respondWithCode.second) + workingCode = getCode(interpreter.getLanguage(), respondWithCode.second) + log.info("Response: \n\t${response.replace("\n", "\n\t", false)}".trimMargin()) + log.info("Code: \n\t${workingCode.replace("\n", "\n\t", false)}".trimMargin()) + } } + return null + } - override fun run(): ExecutionResult { - //language=HTML - log.info("Running ${getCode()}") - OutputInterceptor.clearGlobalOutput() - val result = try { - interpreter.run(getCode()) - } catch (ex: javax.script.ScriptException) { - throw RuntimeException(errorMessage(getCode(), ex.lineNumber, ex.columnNumber, ex.message ?: ""), ex) - } - log.info("Result: $result") - //language=HTML - val executionResult = ExecutionResult(result.toString(), OutputInterceptor.getGlobalOutput()) - OutputInterceptor.clearGlobalOutput() - return executionResult + open fun execute(code: String): ExecutionResult { + //language=HTML + log.info("Running $code") + OutputInterceptor.clearGlobalOutput() + val result = try { + interpreter.run(code) + } catch (ex: ScriptException) { + throw RuntimeException(errorMessage(code, ex.lineNumber, ex.columnNumber, ex.message ?: ""), ex) } + log.info("Result: $result") + //language=HTML + val executionResult = ExecutionResult(result.toString(), OutputInterceptor.getGlobalOutput()) + OutputInterceptor.clearGlobalOutput() + return executionResult } companion object { diff --git a/core/src/main/kotlin/com/simiacryptus/skyenet/core/actors/ImageActor.kt b/core/src/main/kotlin/com/simiacryptus/skyenet/core/actors/ImageActor.kt new file mode 100644 index 00000000..83195376 --- /dev/null +++ b/core/src/main/kotlin/com/simiacryptus/skyenet/core/actors/ImageActor.kt @@ -0,0 +1,52 @@ +package com.simiacryptus.skyenet.core.actors + +import com.simiacryptus.jopenai.API +import com.simiacryptus.jopenai.ApiModel +import com.simiacryptus.jopenai.ApiModel.* +import com.simiacryptus.jopenai.OpenAIClient +import com.simiacryptus.jopenai.models.ChatModels +import com.simiacryptus.jopenai.models.ImageModels +import com.simiacryptus.jopenai.models.OpenAITextModel +import com.simiacryptus.jopenai.proxy.ChatProxy +import java.awt.image.BufferedImage +import java.util.function.Function + +open class ImageActor( + prompt: String = "Transform the user request into an image generation prompt that the user will like", + val action: String? = null, + textModel: ChatModels = ChatModels.GPT35Turbo, + val imageModel: ImageModels = ImageModels.DallE2, + temperature: Double = 0.3, + val width: Int = 1024, + val height: Int = 1024, +) : BaseActor( + prompt = prompt, + name = action, + model = textModel, + temperature = temperature, +) { + private inner class ImageResponseImpl(vararg messages: ChatMessage, val api: API) : ImageResponse { + + private val _text: String by lazy { response(*messages, api = api).choices.first().message?.content ?: throw RuntimeException("No response") } + override fun getText(): String = _text + override fun getImage(): BufferedImage { + val url = (api as OpenAIClient).createImage( + ImageGenerationRequest( + prompt = getText(), + model = imageModel.modelName, + size = "${width}x$height" + ) + ).data.first().url + return javax.imageio.ImageIO.read(java.net.URL(url)) + } + } + + override fun answer(vararg messages: ChatMessage, api: API): ImageResponse { + return ImageResponseImpl(*messages, api = api) + } +} + +interface ImageResponse { + fun getText(): String + fun getImage(): BufferedImage +} \ No newline at end of file diff --git a/core/src/main/kotlin/com/simiacryptus/skyenet/core/actors/ParsedActor.kt b/core/src/main/kotlin/com/simiacryptus/skyenet/core/actors/ParsedActor.kt index 85449fb9..f0a94aa3 100644 --- a/core/src/main/kotlin/com/simiacryptus/skyenet/core/actors/ParsedActor.kt +++ b/core/src/main/kotlin/com/simiacryptus/skyenet/core/actors/ParsedActor.kt @@ -11,7 +11,7 @@ open class ParsedActor( val parserClass: Class>, prompt: String, val action: String? = null, - model: OpenAITextModel = ChatModels.GPT35Turbo, + model: ChatModels = ChatModels.GPT35Turbo, temperature: Double = 0.3, ) : BaseActor>( prompt = prompt, diff --git a/core/src/main/kotlin/com/simiacryptus/skyenet/core/actors/SimpleActor.kt b/core/src/main/kotlin/com/simiacryptus/skyenet/core/actors/SimpleActor.kt index cb4b786e..438a3b23 100644 --- a/core/src/main/kotlin/com/simiacryptus/skyenet/core/actors/SimpleActor.kt +++ b/core/src/main/kotlin/com/simiacryptus/skyenet/core/actors/SimpleActor.kt @@ -7,7 +7,7 @@ import com.simiacryptus.jopenai.models.OpenAITextModel open class SimpleActor( prompt: String, name: String? = null, - model: OpenAITextModel = ChatModels.GPT35Turbo, + model: ChatModels = ChatModels.GPT35Turbo, temperature: Double = 0.3, ) : BaseActor( prompt = prompt, diff --git a/core/src/main/kotlin/com/simiacryptus/skyenet/core/actors/record/CodingActorInterceptor.kt b/core/src/main/kotlin/com/simiacryptus/skyenet/core/actors/record/CodingActorInterceptor.kt index 975f83e2..4e9383ce 100644 --- a/core/src/main/kotlin/com/simiacryptus/skyenet/core/actors/record/CodingActorInterceptor.kt +++ b/core/src/main/kotlin/com/simiacryptus/skyenet/core/actors/record/CodingActorInterceptor.kt @@ -1,15 +1,16 @@ package com.simiacryptus.skyenet.core.actors.record import com.simiacryptus.jopenai.API -import com.simiacryptus.jopenai.OpenAIClient +import com.simiacryptus.jopenai.ApiModel.ChatMessage import com.simiacryptus.jopenai.models.OpenAIModel +import com.simiacryptus.skyenet.core.Brain import com.simiacryptus.skyenet.core.actors.CodeResult import com.simiacryptus.skyenet.core.actors.CodingActor import com.simiacryptus.skyenet.core.util.FunctionWrapper class CodingActorInterceptor( val inner: CodingActor, - val functionInterceptor: FunctionWrapper, + private val functionInterceptor: FunctionWrapper, ) : CodingActor( interpreterClass = inner.interpreterClass, symbols = inner.symbols, @@ -21,32 +22,21 @@ class CodingActorInterceptor( temperature = inner.temperature, autoEvaluate = inner.autoEvaluate, ) { - override fun answer(vararg messages: com.simiacryptus.jopenai.ApiModel.ChatMessage, api: API): CodeResult { - return functionInterceptor.wrap(messages.toList().toTypedArray()) { - messages: Array -> - CodingResultInterceptor(*messages, api = (api as OpenAIClient), inner = inner.answer(*messages, api = api)) + override fun answer(vararg messages: ChatMessage, api: API) = + functionInterceptor.wrap(messages.toList().toTypedArray()) { + inner.answer(*it, api = api) } - } - - private inner class CodingResultInterceptor( - vararg messages: com.simiacryptus.jopenai.ApiModel.ChatMessage, - private val inner: CodeResult, - api: OpenAIClient, - ) : CodeResult { - override fun getStatus() = functionInterceptor.wrap { inner.getStatus() } - override fun getCode() = functionInterceptor.wrap { inner.getCode() } - override fun run() = functionInterceptor.wrap { inner.run() } - - } override fun response( - vararg messages: com.simiacryptus.jopenai.ApiModel.ChatMessage, + vararg messages: ChatMessage, model: OpenAIModel, api: API - ) = functionInterceptor.wrap(messages.toList().toTypedArray(), model) { - messages: Array, + ) = functionInterceptor.wrap( + messages.toList().toTypedArray(), + model + ) { messages: Array, model: OpenAIModel -> - inner.response(*messages, model = model, api = api) + inner.response(*messages, model = model, api = api) } override fun chatMessages(vararg questions: String) = functionInterceptor.wrap(questions) { @@ -59,28 +49,65 @@ class CodingActorInterceptor( override fun answerWithPrefix( codePrefix: String, - vararg messages: com.simiacryptus.jopenai.ApiModel.ChatMessage, + vararg messages: ChatMessage, api: API - ) = functionInterceptor.wrap(messages.toList().toTypedArray(), codePrefix) { - messages: Array, + ) = functionInterceptor.wrap( + messages.toList().toTypedArray(), + codePrefix + ) { messages: Array, codePrefix: String -> - inner.answerWithPrefix(codePrefix, *messages, api = api) + inner.answerWithPrefix(codePrefix, *messages, api = api) } override fun answerWithAutoEval( vararg messages: String, api: API, codePrefix: String - ) = functionInterceptor.wrap(messages.toList().toTypedArray(), codePrefix) { - messages: Array, + ) = functionInterceptor.wrap( + messages.toList().toTypedArray(), + codePrefix + ) { messages: Array, codePrefix: String -> - inner.answerWithAutoEval(*messages, api = api, codePrefix = codePrefix) + inner.answerWithAutoEval(*messages, api = api, codePrefix = codePrefix) } override fun answerWithAutoEval( - vararg messages: com.simiacryptus.jopenai.ApiModel.ChatMessage, + vararg messages: ChatMessage, api: API ) = functionInterceptor.wrap(messages.toList().toTypedArray()) { inner.answerWithAutoEval(*messages, api = api) } + + override fun implement( + self: CodeResult, + brain: Brain, + messages: Array, + codePrefix: String + ) = functionInterceptor.wrap( + messages.toList().toTypedArray(), + codePrefix + ) { messages: Array, + codePrefix: String -> + inner.implement(self, brain, messages, codePrefix) + } + + override fun validateAndFix( + self: CodeResult, + initialCode: String, + codePrefix: String, + brain: Brain, + messages: Array + ) = functionInterceptor.wrap( + messages.toList().toTypedArray(), + initialCode, + codePrefix + ) { messages: Array, + initialCode: String, + codePrefix: String -> + inner.validateAndFix(self, initialCode, codePrefix, brain, messages) ?: "" + } + + override fun execute(code: String) = functionInterceptor.wrap(code) { + inner.execute(it) + } } \ No newline at end of file diff --git a/core/src/main/kotlin/com/simiacryptus/skyenet/core/actors/record/ImageActorInterceptor.kt b/core/src/main/kotlin/com/simiacryptus/skyenet/core/actors/record/ImageActorInterceptor.kt new file mode 100644 index 00000000..67faeb2c --- /dev/null +++ b/core/src/main/kotlin/com/simiacryptus/skyenet/core/actors/record/ImageActorInterceptor.kt @@ -0,0 +1,43 @@ +package com.simiacryptus.skyenet.core.actors.record + +import com.simiacryptus.jopenai.API +import com.simiacryptus.jopenai.models.OpenAIModel +import com.simiacryptus.skyenet.core.actors.ImageActor +import com.simiacryptus.skyenet.core.actors.ParsedResponse +import com.simiacryptus.skyenet.core.util.FunctionWrapper + +class ImageActorInterceptor( + val inner: ImageActor, + private val functionInterceptor: FunctionWrapper, +) : ImageActor( + prompt = inner.prompt, + action = inner.action, + textModel = inner.model, + imageModel = inner.imageModel, + temperature = inner.temperature, + width = inner.width, + height = inner.height, +) { + override fun answer(vararg messages: com.simiacryptus.jopenai.ApiModel.ChatMessage, api: API) = + functionInterceptor.wrap(messages.toList().toTypedArray()) { + inner.answer(*it, api = api) + } + + override fun response( + vararg messages: com.simiacryptus.jopenai.ApiModel.ChatMessage, + model: OpenAIModel, + api: API + ) = functionInterceptor.wrap(messages.toList().toTypedArray(), model) { + messages: Array, + model: OpenAIModel -> + inner.response(*messages, model = model, api = api) + } + + override fun answer(vararg questions: String, api: API) = functionInterceptor.wrap(questions) { + inner.answer(*it, api = api) + } + + override fun chatMessages(vararg questions: String) = functionInterceptor.wrap(questions) { + inner.chatMessages(*it) + } +} \ No newline at end of file diff --git a/core/src/main/kotlin/com/simiacryptus/skyenet/core/actors/record/ParsedActorInterceptor.kt b/core/src/main/kotlin/com/simiacryptus/skyenet/core/actors/record/ParsedActorInterceptor.kt index 004f223a..5209970b 100644 --- a/core/src/main/kotlin/com/simiacryptus/skyenet/core/actors/record/ParsedActorInterceptor.kt +++ b/core/src/main/kotlin/com/simiacryptus/skyenet/core/actors/record/ParsedActorInterceptor.kt @@ -8,7 +8,7 @@ import com.simiacryptus.skyenet.core.util.FunctionWrapper class ParsedActorInterceptor( val inner: ParsedActor, - val functionInterceptor: FunctionWrapper, + private val functionInterceptor: FunctionWrapper, ) : ParsedActor( parserClass = inner.parserClass, prompt = inner.prompt, @@ -16,18 +16,10 @@ class ParsedActorInterceptor( model = inner.model, temperature = inner.temperature, ) { - private inner class ParsedResponseInterceptor(vararg messages: com.simiacryptus.jopenai.ApiModel.ChatMessage, api: API, private val inner: ParsedResponse) : - ParsedResponse(this@ParsedActorInterceptor.inner.resultClass) { - override fun getText() = functionInterceptor.wrap { inner.getText() } - override fun getObj(clazz: Class) = functionInterceptor.intercept(clazz) { inner.getObj(clazz) } // <-- Cannot use 'T' as reified type parameter. Use a class instead. - } - - override fun answer(vararg messages: com.simiacryptus.jopenai.ApiModel.ChatMessage, api: API): ParsedResponse { - return functionInterceptor.wrap(messages.toList().toTypedArray()) { - messages: Array -> - ParsedResponseInterceptor(*messages, api = api, inner = inner.answer(*messages, api = api)) + override fun answer(vararg messages: com.simiacryptus.jopenai.ApiModel.ChatMessage, api: API) = + functionInterceptor.wrap(messages.toList().toTypedArray()) { + inner.answer(*it, api = api) } - } override fun response( vararg messages: com.simiacryptus.jopenai.ApiModel.ChatMessage, diff --git a/core/src/main/kotlin/com/simiacryptus/skyenet/core/actors/test/ImageActorTestBase.kt b/core/src/main/kotlin/com/simiacryptus/skyenet/core/actors/test/ImageActorTestBase.kt new file mode 100644 index 00000000..1660e401 --- /dev/null +++ b/core/src/main/kotlin/com/simiacryptus/skyenet/core/actors/test/ImageActorTestBase.kt @@ -0,0 +1,14 @@ +package com.simiacryptus.skyenet.core.actors.test + +import com.simiacryptus.skyenet.core.actors.BaseActor +import com.simiacryptus.skyenet.core.actors.ImageActor +import com.simiacryptus.skyenet.core.actors.ImageResponse +import com.simiacryptus.skyenet.core.actors.ParsedResponse +import java.util.function.Function + +abstract class ImageActorTestBase() : ActorTestBase() { + override fun actorFactory(prompt: String) = ImageActor( + prompt = prompt, + ) + +} \ No newline at end of file diff --git a/core/src/main/kotlin/com/simiacryptus/skyenet/core/util/FunctionWrapper.kt b/core/src/main/kotlin/com/simiacryptus/skyenet/core/util/FunctionWrapper.kt index ba0d3c3c..457ed578 100644 --- a/core/src/main/kotlin/com/simiacryptus/skyenet/core/util/FunctionWrapper.kt +++ b/core/src/main/kotlin/com/simiacryptus/skyenet/core/util/FunctionWrapper.kt @@ -12,6 +12,11 @@ class FunctionWrapper(val inner: FunctionInterceptor) : FunctionInterceptor { inline fun wrap(p: P, crossinline fn: (P) -> T) = inner.intercept(p, T::class.java) { fn(it) } inline fun wrap(p1: P1, p2: P2, crossinline fn: (P1, P2) -> T) = inner.intercept(p1, p2, T::class.java) { p1, p2 -> fn(p1, p2) } + inline fun wrap(p1: P1, p2: P2, p3: P3, crossinline fn: (P1, P2, P3) -> T) = + inner.intercept(p1, p2, p3, T::class.java) { p1, p2, p3 -> fn(p1, p2, p3) } + + inline fun wrap(p1: P1, p2: P2, p3: P3, p4: P4, crossinline fn: (P1, P2, P3, P4) -> T) = + inner.intercept(p1, p2, p3, p4, T::class.java) { p1, p2, p3, p4 -> fn(p1, p2, p3, p4) } override fun intercept(returnClazz: Class, fn: () -> T) = inner.intercept(returnClazz, fn) @@ -33,6 +38,18 @@ interface FunctionInterceptor { @Suppress("UNCHECKED_CAST") fn(it[0] as P1, it[1] as P2) } + + fun intercept(p1: P1, p2: P2, p3: P3, returnClazz: Class, fn: (P1, P2, P3) -> T) = + intercept(listOf(p1, p2, p3), returnClazz) { + @Suppress("UNCHECKED_CAST") + fn(it[0] as P1, it[1] as P2, it[2] as P3) + } + + fun intercept(p1: P1, p2: P2, p3: P3, p4: P4, returnClazz: Class, fn: (P1, P2, P3, P4) -> T) = + intercept(listOf(p1, p2, p3, p4), returnClazz) { + @Suppress("UNCHECKED_CAST") + fn(it[0] as P1, it[1] as P2, it[2] as P3, it[3] as P4) + } } class NoopFunctionInterceptor : FunctionInterceptor { diff --git a/gradle.properties b/gradle.properties index 4ab4ef01..295bd933 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,6 +1,6 @@ # Gradle Releases -> https://github.com/gradle/gradle/releases libraryGroup = com.simiacryptus.skyenet -libraryVersion = 1.0.37 +libraryVersion = 1.0.38 gradleVersion = 7.6.1 # Opt-out flag for bundling Kotlin standard library -> https://plugins.jetbrains.com/docs/intellij/kotlin.html#kotlin-standard-library diff --git a/webui/build.gradle.kts b/webui/build.gradle.kts index fbed1caf..e4573dab 100644 --- a/webui/build.gradle.kts +++ b/webui/build.gradle.kts @@ -32,7 +32,7 @@ val jetty_version = "11.0.18" val jackson_version = "2.15.3" dependencies { - implementation(group = "com.simiacryptus", name = "jo-penai", version = "1.0.34") + implementation(group = "com.simiacryptus", name = "jo-penai", version = "1.0.35") implementation(project(":core")) testImplementation(project(":groovy")) diff --git a/webui/src/main/kotlin/com/simiacryptus/skyenet/webui/application/ApplicationInterface.kt b/webui/src/main/kotlin/com/simiacryptus/skyenet/webui/application/ApplicationInterface.kt index 3bb9419e..f2c47d0a 100644 --- a/webui/src/main/kotlin/com/simiacryptus/skyenet/webui/application/ApplicationInterface.kt +++ b/webui/src/main/kotlin/com/simiacryptus/skyenet/webui/application/ApplicationInterface.kt @@ -1,6 +1,7 @@ package com.simiacryptus.skyenet.webui.application import com.simiacryptus.skyenet.webui.session.SessionMessage +import com.simiacryptus.skyenet.webui.session.SocketManagerBase import java.util.function.Consumer class ApplicationInterface(private val inner: ApplicationSocketManager) { @@ -11,7 +12,10 @@ class ApplicationInterface(private val inner: ApplicationSocketManager) { fun textInput(handler: Consumer): String = inner.textInput(handler) - fun newMessage(operationID: String, spinner: String, cancelable: Boolean = false): SessionMessage = - inner.newMessage(operationID, spinner, cancelable) + fun newMessage( + operationID: String = SocketManagerBase.randomID(), + spinner: String = SessionMessage.spinner, + cancelable: Boolean = false + ): SessionMessage = inner.newMessage(operationID, spinner, cancelable) } \ No newline at end of file diff --git a/webui/src/main/kotlin/com/simiacryptus/skyenet/webui/application/ApplicationServer.kt b/webui/src/main/kotlin/com/simiacryptus/skyenet/webui/application/ApplicationServer.kt index e87b2db5..8d6fb34b 100644 --- a/webui/src/main/kotlin/com/simiacryptus/skyenet/webui/application/ApplicationServer.kt +++ b/webui/src/main/kotlin/com/simiacryptus/skyenet/webui/application/ApplicationServer.kt @@ -11,6 +11,7 @@ import com.simiacryptus.skyenet.core.platform.Session import com.simiacryptus.skyenet.core.platform.User import com.simiacryptus.skyenet.webui.chat.ChatServer import com.simiacryptus.skyenet.webui.servlet.* +import com.simiacryptus.skyenet.webui.session.SessionMessage import com.simiacryptus.skyenet.webui.session.SocketManager import jakarta.servlet.http.HttpServletRequest import jakarta.servlet.http.HttpServletResponse @@ -114,8 +115,6 @@ abstract class ApplicationServer( companion object { private val log = LoggerFactory.getLogger(ApplicationServer::class.java) - val spinner = - """
Loading...
""" fun getMimeType(filename: String): String = when { diff --git a/webui/src/main/kotlin/com/simiacryptus/skyenet/webui/application/ApplicationSocketManager.kt b/webui/src/main/kotlin/com/simiacryptus/skyenet/webui/application/ApplicationSocketManager.kt index 83585133..6bf3ca28 100644 --- a/webui/src/main/kotlin/com/simiacryptus/skyenet/webui/application/ApplicationSocketManager.kt +++ b/webui/src/main/kotlin/com/simiacryptus/skyenet/webui/application/ApplicationSocketManager.kt @@ -6,6 +6,7 @@ import com.simiacryptus.skyenet.core.platform.DataStorage import com.simiacryptus.skyenet.core.platform.Session import com.simiacryptus.skyenet.core.platform.User import com.simiacryptus.skyenet.webui.chat.ChatSocket +import com.simiacryptus.skyenet.webui.session.SessionMessage import com.simiacryptus.skyenet.webui.session.SocketManagerBase import java.util.function.Consumer @@ -45,6 +46,10 @@ abstract class ApplicationSocketManager( val consumer = linkTriggers[id] consumer ?: throw IllegalArgumentException("No link handler found") consumer.accept(Unit) + } else if (code.startsWith("userTxt,")) { + val consumer = txtTriggers[id] + consumer ?: throw IllegalArgumentException("No input handler found") + consumer.accept(code.removePrefix("userTxt,")) } else { throw IllegalArgumentException("Unknown command: $code") } @@ -75,7 +80,7 @@ abstract class ApplicationSocketManager( ) companion object { - val spinner: String get() = """
${ApplicationServer.spinner}
""" + val spinner: String get() = """
${SessionMessage.spinner}
""" // val playButton: String get() = """""" // val cancelButton: String get() = """""" // val regenButton: String get() = """""" diff --git a/webui/src/main/kotlin/com/simiacryptus/skyenet/webui/chat/ChatSocketManager.kt b/webui/src/main/kotlin/com/simiacryptus/skyenet/webui/chat/ChatSocketManager.kt index 4df72ab5..3a5c2da0 100644 --- a/webui/src/main/kotlin/com/simiacryptus/skyenet/webui/chat/ChatSocketManager.kt +++ b/webui/src/main/kotlin/com/simiacryptus/skyenet/webui/chat/ChatSocketManager.kt @@ -7,6 +7,7 @@ import com.simiacryptus.jopenai.models.ChatModels import com.simiacryptus.jopenai.models.OpenAITextModel import com.simiacryptus.skyenet.core.platform.Session import com.simiacryptus.skyenet.webui.application.ApplicationServer +import com.simiacryptus.skyenet.webui.session.SessionMessage import com.simiacryptus.skyenet.webui.session.SocketManagerBase import com.simiacryptus.skyenet.webui.util.MarkdownUtil @@ -41,7 +42,7 @@ open class ChatSocketManager( override fun onRun(userMessage: String, socket: ChatSocket) { var responseContents = divInitializer(cancelable = false) responseContents += """
${renderResponse(userMessage)}
""" - send("""$responseContents
${ApplicationServer.spinner}
""") + send("""$responseContents
${SessionMessage.spinner}
""") messages += ApiModel.ChatMessage(ApiModel.Role.user, userMessage.toContentList()) try { val response = api.chat( diff --git a/webui/src/main/kotlin/com/simiacryptus/skyenet/webui/session/SessionMessage.kt b/webui/src/main/kotlin/com/simiacryptus/skyenet/webui/session/SessionMessage.kt index 1cc0da5e..cf67186e 100644 --- a/webui/src/main/kotlin/com/simiacryptus/skyenet/webui/session/SessionMessage.kt +++ b/webui/src/main/kotlin/com/simiacryptus/skyenet/webui/session/SessionMessage.kt @@ -1,9 +1,78 @@ package com.simiacryptus.skyenet.webui.session -import com.simiacryptus.skyenet.core.platform.Session +import com.simiacryptus.skyenet.webui.util.MarkdownUtil.renderMarkdown +import org.slf4j.LoggerFactory +import java.awt.image.BufferedImage +import java.util.* -abstract class SessionMessage { - abstract fun append(htmlToAppend: String, showSpinner: Boolean) : Unit - abstract fun sessionID(): Session - abstract fun divID(): String -} \ No newline at end of file +abstract class SessionMessage( + private var buffer: MutableList = mutableListOf(), + private val spinner: String = SessionMessage.spinner +) { + val currentText: String + get() = buffer.filter { it.isNotBlank() }.joinToString("") + + private fun append(htmlToAppend: String, showSpinner: Boolean): StringBuilder? { + val stringBuilder: StringBuilder? + if (htmlToAppend.isNotBlank()) { + stringBuilder = StringBuilder("
$htmlToAppend
") + buffer += stringBuilder + } else { + stringBuilder = null + } + send(currentText + if (showSpinner) "
$spinner
" else "") + return stringBuilder + } + + abstract fun send(html: String) + abstract fun save(file: String, data: ByteArray): String + + fun add( + message: String, + showSpinner: Boolean = true, + tag: String = "div", + className: String = "response-message" + ) = append("""<$tag class="$className">$message""", showSpinner) + + fun echo(message: String, showSpinner: Boolean = true, tag: String = "div") = + add(message, showSpinner, tag, "user-message") + + fun header(message: String, showSpinner: Boolean = true, tag: String = "div") = + add(message, showSpinner, tag, "response-header") + + fun verbose(message: String, showSpinner: Boolean = true, tag: String = "pre") = + add(message, showSpinner, tag, "verbose") + + fun error(message: String, showSpinner: Boolean = false, tag: String = "div") = + add(message, showSpinner, tag, "error") + + fun error(e: Throwable, showSpinner: Boolean = false, tag: String = "div") = + add( + "Error: ${renderMarkdown(e.message ?: "")}", + showSpinner, tag, "error" + ) + + fun complete( + message: String = "", + tag: String = "div", + className: String = "response-message" + ) = append("""<$tag class="$className">$message""", false) + + fun image(image: BufferedImage) = + add("""""") + + companion object { + val log = LoggerFactory.getLogger(SessionMessage::class.java) + + const val spinner = + """
Loading...
""" + + private fun BufferedImage.toPng(): ByteArray { + java.io.ByteArrayOutputStream().use { os -> + javax.imageio.ImageIO.write(this, "png", os) + return os.toByteArray() + } + } + + } +} diff --git a/webui/src/main/kotlin/com/simiacryptus/skyenet/webui/session/SocketManagerBase.kt b/webui/src/main/kotlin/com/simiacryptus/skyenet/webui/session/SocketManagerBase.kt index 260c139d..ae993b3e 100644 --- a/webui/src/main/kotlin/com/simiacryptus/skyenet/webui/session/SocketManagerBase.kt +++ b/webui/src/main/kotlin/com/simiacryptus/skyenet/webui/session/SocketManagerBase.kt @@ -5,8 +5,11 @@ import com.simiacryptus.skyenet.core.platform.* import com.simiacryptus.skyenet.webui.chat.ChatServer import com.simiacryptus.skyenet.webui.chat.ChatSocket import com.simiacryptus.skyenet.webui.util.MarkdownUtil +import org.slf4j.LoggerFactory +import java.net.URL import java.util.concurrent.Executors import java.util.concurrent.atomic.AtomicInteger +import kotlin.io.path.Path abstract class SocketManagerBase( protected val session: Session, @@ -43,26 +46,26 @@ abstract class SocketManagerBase( } fun newMessage( - operationID: String, spinner: String, cancelable: Boolean = false + operationID: String = randomID(), + spinner: String = SessionMessage.spinner, + cancelable: Boolean = false ): SessionMessage { var responseContents = divInitializer(operationID, cancelable) send(responseContents) - return object : SessionMessage() { - override fun append(htmlToAppend: String, showSpinner: Boolean) { - if (htmlToAppend.isNotBlank()) { - responseContents += """
$htmlToAppend
""" - } - val spinner1 = if (showSpinner) """
$spinner
""" else "" - return this@SocketManagerBase.send("""$responseContents$spinner1""") - } - - override fun sessionID(): Session { - return this@SocketManagerBase.session - } + return SessionMessageImpl(responseContents, spinner) + } - override fun divID(): String { - return operationID + inner class SessionMessageImpl( + responseContents: String, + spinner: String = SessionMessage.spinner + ) : SessionMessage(mutableListOf(StringBuilder(responseContents)), spinner) { + override fun send(html: String) = this@SocketManagerBase.send(html) + override fun save(file: String, data: ByteArray): String { + dataStorage?.getSessionDir(user, session)?.let { dir -> + dir.mkdirs() + dir.resolve(file).writeBytes(data) } + return "fileIndex/$session/$file" } } @@ -131,7 +134,7 @@ abstract class SocketManagerBase( ) companion object { - private val log = org.slf4j.LoggerFactory.getLogger(ChatServer::class.java) + private val log = LoggerFactory.getLogger(ChatServer::class.java) fun randomID() = (0..5).map { ('a'..'z').random() }.joinToString("") fun divInitializer(operationID: String = randomID(), cancelable: Boolean): String = diff --git a/webui/src/main/kotlin/com/simiacryptus/skyenet/webui/test/CodingActorTestApp.kt b/webui/src/main/kotlin/com/simiacryptus/skyenet/webui/test/CodingActorTestApp.kt index 3ee391bf..ac627f16 100644 --- a/webui/src/main/kotlin/com/simiacryptus/skyenet/webui/test/CodingActorTestApp.kt +++ b/webui/src/main/kotlin/com/simiacryptus/skyenet/webui/test/CodingActorTestApp.kt @@ -8,7 +8,6 @@ import com.simiacryptus.skyenet.core.platform.Session import com.simiacryptus.skyenet.core.platform.User import com.simiacryptus.skyenet.webui.application.ApplicationInterface import com.simiacryptus.skyenet.webui.application.ApplicationServer -import com.simiacryptus.skyenet.webui.session.SocketManagerBase import com.simiacryptus.skyenet.webui.util.MarkdownUtil.renderMarkdown import org.slf4j.LoggerFactory import java.util.* @@ -28,37 +27,46 @@ open class CodingActorTestApp( ui: ApplicationInterface, api: API ) { - val sessionMessage = ui.newMessage(SocketManagerBase.randomID(), spinner, false) - sessionMessage.append("""
${renderMarkdown(userMessage)}
""", true) - val response = actor.answer(userMessage, api = api) - val canPlay = ApplicationServices.authorizationManager.isAuthorized( - this::class.java, - user, - AuthorizationManager.OperationType.Execute - ) - val playLink = if(!canPlay) "" else { - ui.hrefLink("â–¶", "href-link play-button") { - sessionMessage.append("""
Running...
""", true) - val result = response.run() - sessionMessage.append( + val message = ui.newMessage() + try { + message.echo(renderMarkdown(userMessage)) + val response = actor.answer(userMessage, api = api) + val canPlay = ApplicationServices.authorizationManager.isAuthorized( + this::class.java, + user, + AuthorizationManager.OperationType.Execute + ) + val playLink = if (!canPlay) "" else { + ui.hrefLink("â–¶", "href-link play-button") { + message.add("Running...") + val result = response.run() + message.complete( + """ + |
${result.resultValue}
+ |
${result.resultOutput}
+ """.trimMargin() + ) + } + } + message.complete( + renderMarkdown( """ - |
${result.resultValue}
- |
${result.resultOutput}
- """.trimMargin(), false + |```${actor.interpreter.getLanguage().lowercase(Locale.getDefault())} + |${response.getCode()} + |``` + |$playLink + """.trimMargin().trim() ) - } + ) + } + catch (e: Throwable) { + log.warn("Error", e) + message.error(e) } - sessionMessage.append("""
${ - renderMarkdown(""" - |```${actor.interpreter.getLanguage().lowercase(Locale.getDefault())} - |${response.getCode()} - |``` - |$playLink - """.trimMargin().trim()) - }
""", false) } companion object { private val log = LoggerFactory.getLogger(CodingActorTestApp::class.java) } + } \ No newline at end of file diff --git a/webui/src/main/kotlin/com/simiacryptus/skyenet/webui/test/ImageActorTestApp.kt b/webui/src/main/kotlin/com/simiacryptus/skyenet/webui/test/ImageActorTestApp.kt new file mode 100644 index 00000000..ffdeb2eb --- /dev/null +++ b/webui/src/main/kotlin/com/simiacryptus/skyenet/webui/test/ImageActorTestApp.kt @@ -0,0 +1,52 @@ +package com.simiacryptus.skyenet.webui.test + +import com.simiacryptus.jopenai.API +import com.simiacryptus.skyenet.core.actors.ImageActor +import com.simiacryptus.skyenet.core.platform.Session +import com.simiacryptus.skyenet.core.platform.User +import com.simiacryptus.skyenet.webui.application.ApplicationInterface +import com.simiacryptus.skyenet.webui.application.ApplicationServer +import com.simiacryptus.skyenet.webui.util.MarkdownUtil.renderMarkdown +import org.slf4j.LoggerFactory + +open class ImageActorTestApp( + private val actor: ImageActor, + applicationName: String = "ImageActorTest_" + actor.javaClass.simpleName, + temperature: Double = 0.3, +) : ApplicationServer( + applicationName = applicationName, + temperature = temperature, +) { + + data class Settings( + val actor: ImageActor? = null, + ) + override val settingsClass: Class<*> get() = Settings::class.java + @Suppress("UNCHECKED_CAST") override fun initSettings(session: Session): T? = Settings(actor=actor) as T + + override fun newSession( + session: Session, + user: User?, + userMessage: String, + ui: ApplicationInterface, + api: API + ) { + val message = ui.newMessage() + try { + val actor = getSettings(session, user)?.actor ?: actor + message.echo(renderMarkdown(userMessage)) + val response = actor.answer(userMessage, api = api) + message.verbose(response.getText()) + message.image(response.getImage()) + message.complete() + } catch (e: Throwable) { + log.warn("Error flushing image", e) + message.error(e) + } + } + + companion object { + private val log = LoggerFactory.getLogger(ImageActorTestApp::class.java) + } + +} \ No newline at end of file diff --git a/webui/src/main/kotlin/com/simiacryptus/skyenet/webui/test/ParsedActorTestApp.kt b/webui/src/main/kotlin/com/simiacryptus/skyenet/webui/test/ParsedActorTestApp.kt index d5aa378d..1d0501df 100644 --- a/webui/src/main/kotlin/com/simiacryptus/skyenet/webui/test/ParsedActorTestApp.kt +++ b/webui/src/main/kotlin/com/simiacryptus/skyenet/webui/test/ParsedActorTestApp.kt @@ -7,7 +7,6 @@ import com.simiacryptus.skyenet.core.platform.Session import com.simiacryptus.skyenet.core.platform.User import com.simiacryptus.skyenet.webui.application.ApplicationInterface import com.simiacryptus.skyenet.webui.application.ApplicationServer -import com.simiacryptus.skyenet.webui.session.SocketManagerBase import com.simiacryptus.skyenet.webui.util.MarkdownUtil.renderMarkdown import org.slf4j.LoggerFactory @@ -26,21 +25,24 @@ open class ParsedActorTestApp( ui: ApplicationInterface, api: API ) { - val sessionMessage = ui.newMessage(SocketManagerBase.randomID(), spinner, false) - sessionMessage.append("""
${renderMarkdown(userMessage)}
""", true) - val response = actor.answer(userMessage, api = api) - sessionMessage.append( - """
${ + val message = ui.newMessage() + try { + message.echo(renderMarkdown(userMessage)) + val response = actor.answer(userMessage, api = api) + message.complete( renderMarkdown( """ - |${response.getText()} - |``` - |${JsonUtil.toJson(response.getObj())} - |``` - """.trimMargin().trim() - ) - }
""", false - ) + |${response.getText()} + |``` + |${JsonUtil.toJson(response.getObj())} + |``` + """.trimMargin().trim() + ) + ) + } catch (e: Throwable) { + log.warn("Error", e) + message.error(e) + } } companion object { diff --git a/webui/src/main/kotlin/com/simiacryptus/skyenet/webui/test/SimpleActorTestApp.kt b/webui/src/main/kotlin/com/simiacryptus/skyenet/webui/test/SimpleActorTestApp.kt index d0602026..03df505b 100644 --- a/webui/src/main/kotlin/com/simiacryptus/skyenet/webui/test/SimpleActorTestApp.kt +++ b/webui/src/main/kotlin/com/simiacryptus/skyenet/webui/test/SimpleActorTestApp.kt @@ -6,8 +6,7 @@ import com.simiacryptus.skyenet.core.platform.Session import com.simiacryptus.skyenet.core.platform.User import com.simiacryptus.skyenet.webui.application.ApplicationInterface import com.simiacryptus.skyenet.webui.application.ApplicationServer -import com.simiacryptus.skyenet.webui.session.SocketManagerBase -import com.simiacryptus.skyenet.webui.util.MarkdownUtil +import com.simiacryptus.skyenet.webui.util.MarkdownUtil.renderMarkdown import org.slf4j.LoggerFactory open class SimpleActorTestApp( @@ -32,11 +31,16 @@ open class SimpleActorTestApp( ui: ApplicationInterface, api: API ) { - val sessionMessage = ui.newMessage(SocketManagerBase.randomID(), spinner, false) - val actor = getSettings(session, user)?.actor ?: actor - sessionMessage.append("""
${MarkdownUtil.renderMarkdown(userMessage)}
""", true) - val moderatorResponse = actor.answer(userMessage, api = api) - sessionMessage.append("""
${MarkdownUtil.renderMarkdown(moderatorResponse)}
""", false) + val message = ui.newMessage() + try { + val actor = getSettings(session, user)?.actor ?: actor + message.echo(renderMarkdown(userMessage)) + val response = actor.answer(userMessage, api = api) + message.complete(renderMarkdown(response)) + } catch (e: Throwable) { + log.warn("Error", e) + message.error(e) + } } companion object { diff --git a/webui/src/main/kotlin/com/simiacryptus/skyenet/webui/util/EmbeddingVisualizer.kt b/webui/src/main/kotlin/com/simiacryptus/skyenet/webui/util/TensorflowProjector.kt similarity index 99% rename from webui/src/main/kotlin/com/simiacryptus/skyenet/webui/util/EmbeddingVisualizer.kt rename to webui/src/main/kotlin/com/simiacryptus/skyenet/webui/util/TensorflowProjector.kt index b1acad43..3ea45a77 100644 --- a/webui/src/main/kotlin/com/simiacryptus/skyenet/webui/util/EmbeddingVisualizer.kt +++ b/webui/src/main/kotlin/com/simiacryptus/skyenet/webui/util/TensorflowProjector.kt @@ -9,7 +9,7 @@ import com.simiacryptus.skyenet.core.platform.Session import com.simiacryptus.skyenet.core.platform.User import com.simiacryptus.skyenet.webui.application.ApplicationInterface -class EmbeddingVisualizer( +class TensorflowProjector( val api: API, val dataStorage: DataStorage, val sessionID: Session, diff --git a/webui/src/main/resources/application/chat.css b/webui/src/main/resources/application/chat.css index b7f3994a..997e04c0 100644 --- a/webui/src/main/resources/application/chat.css +++ b/webui/src/main/resources/application/chat.css @@ -64,11 +64,28 @@ body { } .reply-form { - width: 100%; + position: relative; + align-items: flex-start; + left: 0; + right: 0; + top: 0; + display: flex; + margin: 0; + padding: 5px; + flex-shrink: 0; + background-color: var(--primary-bg-color); } .reply-input { + background-color: var(--secondary-bg-color); + border-radius: var(--border-radius); + padding: 10px; + margin-bottom: 10px; + overflow: auto; resize: vertical; + flex: 1; + border: 1px solid #ddd; + box-shadow: var(--box-shadow); } .href-link { diff --git a/webui/src/test/kotlin/com/simiacryptus/skyenet/webui/ActorTestAppServer.kt b/webui/src/test/kotlin/com/simiacryptus/skyenet/webui/ActorTestAppServer.kt index da475630..bd9f972b 100644 --- a/webui/src/test/kotlin/com/simiacryptus/skyenet/webui/ActorTestAppServer.kt +++ b/webui/src/test/kotlin/com/simiacryptus/skyenet/webui/ActorTestAppServer.kt @@ -1,6 +1,7 @@ package com.simiacryptus.skyenet.webui import com.simiacryptus.skyenet.core.actors.CodingActor +import com.simiacryptus.skyenet.core.actors.ImageActor import com.simiacryptus.skyenet.core.actors.ParsedActor import com.simiacryptus.skyenet.core.actors.SimpleActor import com.simiacryptus.skyenet.core.platform.ApplicationServices @@ -11,6 +12,7 @@ import com.simiacryptus.skyenet.groovy.GroovyInterpreter import com.simiacryptus.skyenet.kotlin.KotlinInterpreter import com.simiacryptus.skyenet.scala.ScalaLocalInterpreter import com.simiacryptus.skyenet.webui.test.CodingActorTestApp +import com.simiacryptus.skyenet.webui.test.ImageActorTestApp import com.simiacryptus.skyenet.webui.test.ParsedActorTestApp import com.simiacryptus.skyenet.webui.test.SimpleActorTestApp import java.util.function.Function @@ -30,6 +32,7 @@ object ActorTestAppServer : com.simiacryptus.skyenet.webui.application.Applicati listOf( ChildWebApp("/test_simple", SimpleActorTestApp(SimpleActor("Translate the user's request into pig latin.", "PigLatin"))), ChildWebApp("/test_parsed_joke", ParsedActorTestApp(ParsedActor(JokeParser::class.java, "Tell me a joke"))), + ChildWebApp("/images", ImageActorTestApp(ImageActor())), ChildWebApp("/test_coding_scala", CodingActorTestApp(CodingActor(ScalaLocalInterpreter::class))), ChildWebApp("/test_coding_kotlin", CodingActorTestApp(CodingActor(KotlinInterpreter::class))), ChildWebApp("/test_coding_groovy", CodingActorTestApp(CodingActor(GroovyInterpreter::class))),