From 70214fffac84a67bdb4d99803bccfcf1c231038f Mon Sep 17 00:00:00 2001 From: Andrew Charneski Date: Sun, 13 Oct 2024 15:17:55 -0400 Subject: [PATCH] 1.2.10 (#110) * 1.2.10 * misc --- core/build.gradle.kts | 2 +- gradle.properties | 2 +- webui/build.gradle.kts | 2 +- .../simiacryptus/diff/IterativePatchUtil.kt | 4 +- .../skyenet/apps/general/CmdPatchApp.kt | 2 + .../skyenet/apps/parsers/DocumentRecord.kt | 65 +++------- .../skyenet/apps/plan/CommandAutoFixTask.kt | 114 ++++++++++-------- .../skyenet/apps/plan/PlanSettings.kt | 2 +- .../apps/plan/file/EmbeddingSearchTask.kt | 1 - .../simiacryptus/skyenet/util/MarkdownUtil.kt | 2 +- 10 files changed, 86 insertions(+), 110 deletions(-) diff --git a/core/build.gradle.kts b/core/build.gradle.kts index e27b0704..77864b1d 100644 --- a/core/build.gradle.kts +++ b/core/build.gradle.kts @@ -33,7 +33,7 @@ val hsqldb_version = "2.7.2" dependencies { - implementation(group = "com.simiacryptus", name = "jo-penai", version = "1.1.7") + implementation(group = "com.simiacryptus", name = "jo-penai", version = "1.1.9") implementation(group = "org.hsqldb", name = "hsqldb", version = hsqldb_version) implementation("org.apache.commons:commons-text:1.11.0") diff --git a/gradle.properties b/gradle.properties index c5aeba91..9e8a4b7e 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,5 +1,5 @@ # Gradle Releases -> https://github.com/gradle/gradle/releases libraryGroup = com.simiacryptus.skyenet -libraryVersion = 1.2.9 +libraryVersion = 1.2.10 gradleVersion = 7.6.1 kotlin.daemon.jvmargs=-Xmx4g diff --git a/webui/build.gradle.kts b/webui/build.gradle.kts index 9e3d0d45..5b5fdb2a 100644 --- a/webui/build.gradle.kts +++ b/webui/build.gradle.kts @@ -36,7 +36,7 @@ val jackson_version = "2.17.2" dependencies { - implementation(group = "com.simiacryptus", name = "jo-penai", version = "1.1.7") { + implementation(group = "com.simiacryptus", name = "jo-penai", version = "1.1.9") { exclude(group = "org.slf4j") } diff --git a/webui/src/main/kotlin/com/simiacryptus/diff/IterativePatchUtil.kt b/webui/src/main/kotlin/com/simiacryptus/diff/IterativePatchUtil.kt index 8ef79c7c..84f3d8b7 100644 --- a/webui/src/main/kotlin/com/simiacryptus/diff/IterativePatchUtil.kt +++ b/webui/src/main/kotlin/com/simiacryptus/diff/IterativePatchUtil.kt @@ -651,8 +651,8 @@ private fun generatePatchedText( } }, type = when { - line.startsWith("+") -> ADD - line.startsWith("-") -> DELETE + line.trimStart().startsWith("+") -> ADD + line.trimStart().startsWith("-") -> DELETE else -> CONTEXT } ) diff --git a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/general/CmdPatchApp.kt b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/general/CmdPatchApp.kt index 432af83a..bd392cc2 100644 --- a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/general/CmdPatchApp.kt +++ b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/general/CmdPatchApp.kt @@ -100,6 +100,8 @@ class CmdPatchApp( val command = listOf(settings.executable.absolutePath) + settings.arguments.split(" ").filter(String::isNotBlank) val processBuilder = ProcessBuilder(command).directory(settings.workingDirectory) + // Pass the current environment to the subprocess + processBuilder.environment().putAll(System.getenv()) val buffer = StringBuilder() val taskOutput = task.add("") val process = processBuilder.start() diff --git a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/parsers/DocumentRecord.kt b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/parsers/DocumentRecord.kt index 3a96076b..83dbb7bc 100644 --- a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/parsers/DocumentRecord.kt +++ b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/parsers/DocumentRecord.kt @@ -15,59 +15,34 @@ import java.util.concurrent.ExecutorService import java.util.concurrent.TimeUnit data class DocumentRecord( - val id: String, - val parentId: String?, - val type: String, val text: String?, - val tags: String?, + val metadata: String?, val sourcePath: String, - val depth: Int, val jsonPath: String, var vector: DoubleArray?, - val properties: String?, - val relations: String? ) : Serializable { @Throws(IOException::class) fun writeObject(out: ObjectOutputStream) { - out.writeUTF(id) - out.writeObject(parentId) - out.writeUTF(type) - out.writeObject(text) - out.writeObject(tags) + out.writeUTF(text ?: "") + out.writeUTF(metadata ?: "") out.writeUTF(sourcePath) - out.writeInt(depth) out.writeUTF(jsonPath) out.writeObject(vector) - out.writeObject(properties) - out.writeObject(relations) } @Throws(IOException::class, ClassNotFoundException::class) fun readObject(input: ObjectInputStream): DocumentRecord { - val id = input.readUTF() - val parentId = input.readObject() as String? - val type = input.readUTF() - val text = input.readObject() as String? - val entities = input.readObject() as String? - val tags = input.readObject() as String? + val text = input.readUTF().let { if (it.isEmpty()) null else it } + val metadata = input.readUTF().let { if (it.isEmpty()) null else it } val sourcePath = input.readUTF() - val depth = input.readInt() val jsonPath = input.readUTF() val vector = input.readObject() as DoubleArray - val properties = input.readObject() as String? - val relations = input.readObject() as String? return DocumentRecord( - id, - parentId, - type, text, - tags, + metadata, sourcePath, - depth, jsonPath, - vector, - properties, - relations + vector ) } @@ -100,28 +75,22 @@ data class DocumentRecord( openAIClient: OpenAIClient, pool: ExecutorService ) { - fun processContent(content: Map, parentId: String? = null, depth: Int = 0, path: String = "") { + fun processContent(content: Map, path: String = "") { val record = DocumentRecord( - id = content.hashCode().toString(), - parentId = parentId, - type = content["type"] as? String ?: "", text = content["text"] as? String, - tags = (content["tags"] as? List<*>)?.joinToString(","), + metadata = JsonUtil.toJson(content.filter { it.key != "text" && it.key != "content" }), sourcePath = inputPath, - depth = depth, jsonPath = path, - vector = null, - properties = null, - relations = null + vector = null ) records.add(record) (content["content"] as? List>)?.forEachIndexed { index, childContent -> - processContent(childContent, content.hashCode().toString(), depth + 1, "$path.content[$index]") + processContent(childContent, "$path.content[$index]") } } (document as? Map)?.get("content")?.let { contentList -> (contentList as? List>)?.forEachIndexed { index, content -> - processContent(content, null, 0, "content[$index]") + processContent(content, "content[$index]") } } addEmbeddings(records, pool, openAIClient) @@ -169,17 +138,11 @@ data class DocumentRecord( repeat(size) { records.add( DocumentRecord( - id = "", - parentId = null, - type = "", text = null, - tags = null, + metadata = null, sourcePath = "", - depth = 0, jsonPath = "", - vector = DoubleArray(0), - properties = null, - relations = null + vector = DoubleArray(0) ).readObject(input) ) } diff --git a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/CommandAutoFixTask.kt b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/CommandAutoFixTask.kt index 1a9315cd..c2e78bb7 100644 --- a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/CommandAutoFixTask.kt +++ b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/CommandAutoFixTask.kt @@ -12,14 +12,15 @@ import com.simiacryptus.skyenet.webui.session.SessionTask import org.slf4j.LoggerFactory import java.io.File import java.util.concurrent.Semaphore +import java.util.concurrent.atomic.AtomicBoolean class CommandAutoFixTask( planSettings: PlanSettings, planTask: CommandAutoFixTaskData? ) : AbstractTask(planSettings, planTask) { class CommandAutoFixTaskData( - @Description("The command to be executed") - val command: List? = null, + @Description("The commands to be executed") + val commands: List>? = null, @Description("The working directory for the command execution") val workingDir: String? = null, task_description: String? = null, @@ -35,10 +36,10 @@ class CommandAutoFixTask( override fun promptSegment(): String { return """ CommandAutoFix - Run a command and automatically fix any issues that arise - ** Specify the command to be executed and any additional instructions + ** Specify the commands to be executed and any additional instructions ** Specify the working directory relative to the root directory - ** Provide the command and its arguments in the 'command' field - ** List input files/tasks to be examined when fixing issues + ** Provide the commands and their arguments in the 'commands' field + ** Each command should be a list of strings ** Available commands: ${planSettings.commandAutoFixCommands?.joinToString("\n") { " * ${File(it).name}" }} """.trimMargin() @@ -55,61 +56,72 @@ CommandAutoFix - Run a command and automatically fix any issues that arise resultFn: (String) -> Unit ) { val semaphore = Semaphore(0) + val hasError = AtomicBoolean(false) val onComplete = { semaphore.release() } Retryable(agent.ui, task = task) { val task = agent.ui.newTask(false).apply { it.append(placeholder) } - val alias = this.planTask?.command?.first() - val commandAutoFixCommands = agent.planSettings.commandAutoFixCommands - val cmds = commandAutoFixCommands - ?.map { File(it) }?.associateBy { it.name } - ?.filterKeys { it.startsWith(alias ?: "") } - ?: emptyMap() - val executable = cmds.entries.firstOrNull()?.value - if (executable == null) { - throw IllegalArgumentException("Command not found: $alias") - } - val workingDirectory = (this.planTask?.workingDir - ?.let { agent.root.toFile().resolve(it) } ?: agent.root.toFile()) - .apply { mkdirs() } - val outputResult = CmdPatchApp( - root = agent.root, - session = agent.session, - settings = PatchApp.Settings( - executable = executable, - arguments = this.planTask?.command?.drop(1)?.joinToString(" ") ?: "", - workingDirectory = workingDirectory, - exitCodeOption = "nonzero", - additionalInstructions = "", - autoFix = agent.planSettings.autoFix - ), - api = api as ChatClient, - files = agent.files, - model = agent.planSettings.getTaskSettings(TaskType.valueOf(planTask?.task_type!!)).model - ?: agent.planSettings.defaultModel, - ).run( - ui = agent.ui, - task = task - ) - resultFn("Command Auto Fix completed") - task.add(if (outputResult.exitCode == 0) { - if (agent.planSettings.autoFix) { - onComplete() - MarkdownUtil.renderMarkdown("## Auto-applied Command Auto Fix\n", ui = agent.ui) + this.planTask?.commands?.forEachIndexed { index, command -> + val alias = command.firstOrNull() + val commandAutoFixCommands = agent.planSettings.commandAutoFixCommands + val cmds = commandAutoFixCommands + ?.map { File(it) }?.associateBy { it.name } + ?.filterKeys { it.startsWith(alias ?: "") } + ?: emptyMap() + val executable = cmds.entries.firstOrNull()?.value + if (executable == null) { + throw IllegalArgumentException("Command not found: $alias") + } + val workingDirectory = (this.planTask.workingDir + ?.let { agent.root.toFile().resolve(it) } ?: agent.root.toFile()) + .apply { mkdirs() } + val outputResult = CmdPatchApp( + root = agent.root, + session = agent.session, + settings = PatchApp.Settings( + executable = executable, + arguments = command.drop(1).joinToString(" "), + workingDirectory = workingDirectory, + exitCodeOption = "nonzero", + additionalInstructions = "", + autoFix = agent.planSettings.autoFix + ), + api = api as ChatClient, + files = agent.files, + model = agent.planSettings.getTaskSettings(TaskType.valueOf(planTask.task_type!!)).model + ?: agent.planSettings.defaultModel, + ).run( + ui = agent.ui, + task = task + ) + if (outputResult.exitCode != 0) { + hasError.set(true) + } + task.add(MarkdownUtil.renderMarkdown("## Command Auto Fix Result for Command ${index + 1}\n", ui = agent.ui, tabs = false)) + task.add(if (outputResult.exitCode == 0) { + if (agent.planSettings.autoFix) { + MarkdownUtil.renderMarkdown("Auto-applied Command Auto Fix\n", ui = agent.ui, tabs = false) + } else { + MarkdownUtil.renderMarkdown( + "Command Auto Fix Result\n", + ui = agent.ui, tabs = false + ) + } } else { MarkdownUtil.renderMarkdown( - "## Command Auto Fix Result\n", - ui = agent.ui - ) + acceptButtonFooter( - agent.ui - ) { - onComplete() - } - } + "Command Auto Fix Failed\n", + ui = agent.ui, tabs = false + ) + }) + } + resultFn("All Command Auto Fix tasks completed") + task.add(if (!hasError.get()) { + onComplete() + MarkdownUtil.renderMarkdown("## All Command Auto Fix tasks completed successfully\n", ui = agent.ui, tabs = false) } else { MarkdownUtil.renderMarkdown( - "## Command Auto Fix Failed\n", + "## Some Command Auto Fix tasks failed\n", ui = agent.ui ) + acceptButtonFooter( agent.ui diff --git a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/PlanSettings.kt b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/PlanSettings.kt index f854ac1d..43037cf1 100644 --- a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/PlanSettings.kt +++ b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/PlanSettings.kt @@ -137,7 +137,7 @@ ${taskType.name}: "1" to CommandAutoFixTaskData( task_description = "Task 1", task_dependencies = listOf(), - command = listOf("npx", "create-react-app", ".", "--template", "typescript"), + commands = listOf(listOf("npx", "create-react-app", ".", "--template", "typescript")), workingDir = "." ), "2" to FileModificationTaskData( diff --git a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/file/EmbeddingSearchTask.kt b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/file/EmbeddingSearchTask.kt index 07fed4b9..40c5b95e 100644 --- a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/file/EmbeddingSearchTask.kt +++ b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/file/EmbeddingSearchTask.kt @@ -104,7 +104,6 @@ EmbeddingSearch - Search for similar embeddings in index files and provide top r appendLine("- File: ${result.file}") appendLine("- Distance: ${result.distance}") appendLine("- Text: ${result.record.text}") - appendLine("- Type: ${result.record.type}") appendLine("- Source Path: ${result.record.sourcePath}") appendLine("- JSON Path: ${result.record.jsonPath}") appendLine() diff --git a/webui/src/main/kotlin/com/simiacryptus/skyenet/util/MarkdownUtil.kt b/webui/src/main/kotlin/com/simiacryptus/skyenet/util/MarkdownUtil.kt index 2f8990a1..f51892ed 100644 --- a/webui/src/main/kotlin/com/simiacryptus/skyenet/util/MarkdownUtil.kt +++ b/webui/src/main/kotlin/com/simiacryptus/skyenet/util/MarkdownUtil.kt @@ -73,7 +73,7 @@ object MarkdownUtil { } else htmlContent } - var MMDC_CMD: List = listOf("powershell", "mmdc") + var MMDC_CMD: List = listOf("mmdc") private fun renderMermaidToSVG(mermaidCode: String): String { // mmdc -i input.mmd -o output.svg val tempInputFile = Files.createTempFile("mermaid", ".mmd").toFile()