From 583970c33c3469c81f6d074851e0207bc63436db Mon Sep 17 00:00:00 2001 From: Andrew Charneski Date: Wed, 11 Dec 2024 21:43:10 -0500 Subject: [PATCH] wip --- .../skyenet/apps/general/AutoPlanChatApp.kt | 2 +- .../skyenet/apps/general/PlanChatApp.kt | 2 +- .../skyenet/apps/plan/PlanSettings.kt | 9 +- .../skyenet/apps/plan/TaskType.kt | 36 +++-- .../apps/plan/WebFetchAndTransformTask.kt | 144 ----------------- .../apps/plan/file/TestGenerationTask.kt | 95 ----------- .../plan/{ => tools}/CommandAutoFixTask.kt | 6 +- .../plan/{ => tools}/CommandSessionTask.kt | 37 ++--- .../apps/plan/{ => tools}/ForeachTask.kt | 20 ++- .../apps/plan/{ => tools}/GitHubSearchTask.kt | 3 +- .../apps/plan/{ => tools}/GoogleSearchTask.kt | 3 +- .../apps/plan/{ => tools}/PlanningTask.kt | 25 ++- .../plan/{ => tools}/RunShellCommandTask.kt | 8 +- .../apps/plan/{ => tools}/SearchTask.kt | 5 +- .../plan/{ => tools}/SeleniumSessionTask.kt | 10 +- .../plan/tools/WebFetchAndTransformTask.kt | 147 ++++++++++++++++++ .../{ => tools}/file/AbstractAnalysisTask.kt | 2 +- .../plan/{ => tools}/file/AbstractFileTask.kt | 4 +- .../{ => tools}/file/CodeOptimizationTask.kt | 4 +- .../plan/{ => tools}/file/CodeReviewTask.kt | 4 +- .../{ => tools}/file/DocumentationTask.kt | 4 +- .../{ => tools}/file/FileModificationTask.kt | 4 +- .../apps/plan/{ => tools}/file/InquiryTask.kt | 2 +- .../file/PerformanceAnalysisTask.kt | 4 +- .../plan/{ => tools}/file/RefactorTask.kt | 4 +- .../{ => tools}/file/SecurityAuditTask.kt | 4 +- .../plan/tools/file/TestGenerationTask.kt | 94 +++++++++++ .../knowledge/EmbeddingSearchTask.kt | 2 +- .../knowledge/KnowledgeIndexingTask.kt | 2 +- .../knowledge/WebSearchAndIndexTask.kt | 2 +- 30 files changed, 345 insertions(+), 343 deletions(-) delete mode 100644 webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/file/TestGenerationTask.kt rename webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/{ => tools}/CommandAutoFixTask.kt (97%) rename webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/{ => tools}/CommandSessionTask.kt (89%) rename webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/{ => tools}/ForeachTask.kt (71%) rename webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/{ => tools}/GitHubSearchTask.kt (98%) rename webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/{ => tools}/GoogleSearchTask.kt (97%) rename webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/{ => tools}/PlanningTask.kt (85%) rename webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/{ => tools}/RunShellCommandTask.kt (95%) rename webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/{ => tools}/SearchTask.kt (97%) rename webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/{ => tools}/SeleniumSessionTask.kt (96%) create mode 100644 webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/tools/WebFetchAndTransformTask.kt rename webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/{ => tools}/file/AbstractAnalysisTask.kt (98%) rename webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/{ => tools}/file/AbstractFileTask.kt (94%) rename webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/{ => tools}/file/CodeOptimizationTask.kt (93%) rename webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/{ => tools}/file/CodeReviewTask.kt (93%) rename webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/{ => tools}/file/DocumentationTask.kt (97%) rename webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/{ => tools}/file/FileModificationTask.kt (97%) rename webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/{ => tools}/file/InquiryTask.kt (99%) rename webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/{ => tools}/file/PerformanceAnalysisTask.kt (93%) rename webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/{ => tools}/file/RefactorTask.kt (94%) rename webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/{ => tools}/file/SecurityAuditTask.kt (93%) create mode 100644 webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/tools/file/TestGenerationTask.kt rename webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/{ => tools}/knowledge/EmbeddingSearchTask.kt (99%) rename webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/{ => tools}/knowledge/KnowledgeIndexingTask.kt (98%) rename webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/{ => tools}/knowledge/WebSearchAndIndexTask.kt (98%) diff --git a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/general/AutoPlanChatApp.kt b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/general/AutoPlanChatApp.kt index b4b16144..41f06c93 100644 --- a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/general/AutoPlanChatApp.kt +++ b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/general/AutoPlanChatApp.kt @@ -7,7 +7,7 @@ import com.simiacryptus.jopenai.describe.Description import com.simiacryptus.jopenai.models.ChatModel import com.simiacryptus.skyenet.TabbedDisplay import com.simiacryptus.skyenet.apps.plan.* -import com.simiacryptus.skyenet.apps.plan.file.FileModificationTask.FileModificationTaskConfigData +import com.simiacryptus.skyenet.apps.plan.tools.file.FileModificationTask.FileModificationTaskConfigData import com.simiacryptus.skyenet.core.actors.ParsedActor import com.simiacryptus.skyenet.core.platform.Session import com.simiacryptus.skyenet.core.platform.model.User diff --git a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/general/PlanChatApp.kt b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/general/PlanChatApp.kt index d8ac55ab..7b60dd72 100644 --- a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/general/PlanChatApp.kt +++ b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/general/PlanChatApp.kt @@ -5,7 +5,7 @@ import com.simiacryptus.jopenai.ChatClient import com.simiacryptus.jopenai.OpenAIClient import com.simiacryptus.jopenai.models.ChatModel import com.simiacryptus.skyenet.apps.plan.* -import com.simiacryptus.skyenet.apps.plan.file.InquiryTask.InquiryTaskConfigData +import com.simiacryptus.skyenet.apps.plan.tools.file.InquiryTask.InquiryTaskConfigData import com.simiacryptus.skyenet.core.platform.Session import com.simiacryptus.skyenet.core.platform.model.User import com.simiacryptus.skyenet.util.MarkdownUtil diff --git a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/PlanSettings.kt b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/PlanSettings.kt index 01ee743c..665eb79a 100644 --- a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/PlanSettings.kt +++ b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/PlanSettings.kt @@ -2,13 +2,14 @@ package com.simiacryptus.skyenet.apps.plan import com.simiacryptus.jopenai.describe.AbbrevWhitelistYamlDescriber import com.simiacryptus.jopenai.models.ChatModel -import com.simiacryptus.skyenet.apps.plan.CommandAutoFixTask.CommandAutoFixTaskConfigData +import com.simiacryptus.skyenet.apps.plan.tools.CommandAutoFixTask.CommandAutoFixTaskConfigData import com.simiacryptus.skyenet.apps.plan.PlanUtil.isWindows -import com.simiacryptus.skyenet.apps.plan.PlanningTask.PlanningTaskConfigData -import com.simiacryptus.skyenet.apps.plan.PlanningTask.TaskBreakdownResult +import com.simiacryptus.skyenet.apps.plan.tools.PlanningTask.PlanningTaskConfigData +import com.simiacryptus.skyenet.apps.plan.tools.PlanningTask.TaskBreakdownResult import com.simiacryptus.skyenet.apps.plan.TaskType.Companion.getAvailableTaskTypes import com.simiacryptus.skyenet.apps.plan.TaskType.Companion.getImpl -import com.simiacryptus.skyenet.apps.plan.file.FileModificationTask.FileModificationTaskConfigData +import com.simiacryptus.skyenet.apps.plan.tools.file.FileModificationTask.FileModificationTaskConfigData +import com.simiacryptus.skyenet.apps.plan.tools.CommandAutoFixTask import com.simiacryptus.skyenet.core.actors.ParsedActor diff --git a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/TaskType.kt b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/TaskType.kt index 05bffda2..132d8586 100644 --- a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/TaskType.kt +++ b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/TaskType.kt @@ -2,24 +2,26 @@ package com.simiacryptus.skyenet.apps.plan import com.fasterxml.jackson.databind.annotation.JsonDeserialize import com.fasterxml.jackson.databind.annotation.JsonSerialize -import com.simiacryptus.skyenet.apps.plan.CommandAutoFixTask.CommandAutoFixTaskConfigData -import com.simiacryptus.skyenet.apps.plan.ForeachTask.ForeachTaskConfigData -import com.simiacryptus.skyenet.apps.plan.GoogleSearchTask.GoogleSearchTaskConfigData -import com.simiacryptus.skyenet.apps.plan.PlanningTask.PlanningTaskConfigData -import com.simiacryptus.skyenet.apps.plan.RunShellCommandTask.RunShellCommandTaskConfigData +import com.simiacryptus.skyenet.apps.plan.tools.CommandAutoFixTask.CommandAutoFixTaskConfigData +import com.simiacryptus.skyenet.apps.plan.tools.ForeachTask.ForeachTaskConfigData +import com.simiacryptus.skyenet.apps.plan.tools.GoogleSearchTask.GoogleSearchTaskConfigData +import com.simiacryptus.skyenet.apps.plan.tools.PlanningTask.PlanningTaskConfigData +import com.simiacryptus.skyenet.apps.plan.tools.RunShellCommandTask.RunShellCommandTaskConfigData import com.simiacryptus.skyenet.apps.plan.file.* -import com.simiacryptus.skyenet.apps.plan.file.CodeOptimizationTask.CodeOptimizationTaskConfigData -import com.simiacryptus.skyenet.apps.plan.file.CodeReviewTask.CodeReviewTaskConfigData -import com.simiacryptus.skyenet.apps.plan.file.DocumentationTask.DocumentationTaskConfigData -import com.simiacryptus.skyenet.apps.plan.file.FileModificationTask.FileModificationTaskConfigData -import com.simiacryptus.skyenet.apps.plan.file.InquiryTask.InquiryTaskConfigData -import com.simiacryptus.skyenet.apps.plan.file.PerformanceAnalysisTask.PerformanceAnalysisTaskConfigData -import com.simiacryptus.skyenet.apps.plan.file.RefactorTask.RefactorTaskConfigData -import com.simiacryptus.skyenet.apps.plan.file.SecurityAuditTask.SecurityAuditTaskConfigData -import com.simiacryptus.skyenet.apps.plan.file.TestGenerationTask.TestGenerationTaskConfigData -import com.simiacryptus.skyenet.apps.plan.knowledge.EmbeddingSearchTask -import com.simiacryptus.skyenet.apps.plan.knowledge.KnowledgeIndexingTask -import com.simiacryptus.skyenet.apps.plan.knowledge.WebSearchAndIndexTask +import com.simiacryptus.skyenet.apps.plan.tools.file.CodeOptimizationTask.CodeOptimizationTaskConfigData +import com.simiacryptus.skyenet.apps.plan.tools.file.CodeReviewTask.CodeReviewTaskConfigData +import com.simiacryptus.skyenet.apps.plan.tools.file.DocumentationTask.DocumentationTaskConfigData +import com.simiacryptus.skyenet.apps.plan.tools.file.FileModificationTask.FileModificationTaskConfigData +import com.simiacryptus.skyenet.apps.plan.tools.file.InquiryTask.InquiryTaskConfigData +import com.simiacryptus.skyenet.apps.plan.tools.file.PerformanceAnalysisTask.PerformanceAnalysisTaskConfigData +import com.simiacryptus.skyenet.apps.plan.tools.file.RefactorTask.RefactorTaskConfigData +import com.simiacryptus.skyenet.apps.plan.tools.file.SecurityAuditTask.SecurityAuditTaskConfigData +import com.simiacryptus.skyenet.apps.plan.tools.file.TestGenerationTask.TestGenerationTaskConfigData +import com.simiacryptus.skyenet.apps.plan.tools.knowledge.EmbeddingSearchTask +import com.simiacryptus.skyenet.apps.plan.tools.knowledge.KnowledgeIndexingTask +import com.simiacryptus.skyenet.apps.plan.tools.knowledge.WebSearchAndIndexTask +import com.simiacryptus.skyenet.apps.plan.tools.* +import com.simiacryptus.skyenet.apps.plan.tools.file.* import com.simiacryptus.util.DynamicEnum import com.simiacryptus.util.DynamicEnumDeserializer import com.simiacryptus.util.DynamicEnumSerializer diff --git a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/WebFetchAndTransformTask.kt b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/WebFetchAndTransformTask.kt index 75647e37..e4450783 100644 --- a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/WebFetchAndTransformTask.kt +++ b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/WebFetchAndTransformTask.kt @@ -1,150 +1,6 @@ package com.simiacryptus.skyenet.apps.plan -import com.simiacryptus.jopenai.API -import com.simiacryptus.jopenai.ChatClient -import com.simiacryptus.jopenai.OpenAIClient -import com.simiacryptus.jopenai.describe.Description -import com.simiacryptus.skyenet.core.actors.SimpleActor -import com.simiacryptus.skyenet.util.MarkdownUtil -import com.simiacryptus.skyenet.webui.session.SessionTask -import org.apache.hc.client5.http.classic.methods.HttpGet -import org.apache.hc.client5.http.impl.classic.HttpClients -import org.apache.hc.core5.http.io.entity.EntityUtils -import org.jsoup.Jsoup -import org.jsoup.nodes.Document import org.jsoup.nodes.Node -import org.jsoup.select.NodeFilter.FilterResult -import org.slf4j.LoggerFactory - -open class WebFetchAndTransformTask( - planSettings: PlanSettings, - planTask: WebFetchAndTransformTaskConfigData? -) : AbstractTask(planSettings, planTask) { - class WebFetchAndTransformTaskConfigData( - @Description("The URL to fetch") - val url: String, - @Description("The desired format or focus for the transformation") - val transformationGoal: String, - task_description: String? = null, - task_dependencies: List? = null, - state: TaskState? = null, - ) : TaskConfigBase( - task_type = TaskType.WebFetchAndTransform.name, - task_description = task_description, - task_dependencies = task_dependencies, - state = state - ) - - override fun promptSegment() = """ - WebFetchAndTransform - Fetch a web page, strip HTML, and transform content - ** Specify the URL to fetch - ** Specify the desired format or focus for the transformation - """.trimMargin() - - override fun run( - agent: PlanCoordinator, - messages: List, - task: SessionTask, - api: ChatClient, - resultFn: (String) -> Unit, - api2: OpenAIClient, - planSettings: PlanSettings - ) { - val fetchedContent = fetchAndStripHtml(taskConfig?.url ?: "") - val transformedContent = transformContent(fetchedContent, taskConfig?.transformationGoal ?: "", api, planSettings) - task.add(MarkdownUtil.renderMarkdown(transformedContent, ui = agent.ui)) - resultFn(transformedContent) - } - - private fun fetchAndStripHtml(url: String): String { - HttpClients.createDefault().use { httpClient -> - val httpGet = HttpGet(url) - httpClient.execute(httpGet).use { response -> - val entity = response.entity - val content = EntityUtils.toString(entity) - return scrubHtml(content) - } - } - } - - private fun transformContent(content: String, transformationGoal: String, api: API, planSettings: PlanSettings): String { - val prompt = """ - Transform the following web content according to this goal: $transformationGoal - - Content: - $content - - Transformed content: - """.trimIndent() - return SimpleActor( - prompt = prompt, - model = planSettings.defaultModel, - ).answer( - listOf( - """ - |Transform the following web content according to this goal: $transformationGoal - | - |$content - """.trimMargin(), - ), api - ) - } - - companion object { - private val log = LoggerFactory.getLogger(WebFetchAndTransformTask::class.java) - fun scrubHtml(str: String, maxLength: Int = 100 * 1024): String { - val document: Document = Jsoup.parse(str) - // Remove unnecessary elements, attributes, and optimize the document - document.apply { - if (document.body().html().length > maxLength) return@apply - select("script, style, link, meta, iframe, noscript").remove() // Remove unnecessary and potentially harmful tags - outputSettings().prettyPrint(false) // Disable pretty printing for compact output - if (document.body().html().length > maxLength) return@apply - // Remove comments - select("*").forEach { it.childNodes().removeAll { node -> node.nodeName() == "#comment" } } - if (document.body().html().length > maxLength) return@apply - // Remove data-* attributes - select("*[data-*]").forEach { it.attributes().removeAll { attr -> attr.key.startsWith("data-") } } - if (document.body().html().length > maxLength) return@apply - select("*").forEach { element -> - val importantAttributes = setOf("href", "src", "alt", "title", "width", "height", "style", "class", "id", "name") - element.attributes().removeAll { it.key !in importantAttributes } - } - if (document.body().html().length > maxLength) return@apply - // Remove empty elements - select("*").filter { node, depth -> - if(node.text().isBlank() && node.attributes().isEmpty() && !node.hasAttr("img")) FilterResult.REMOVE else FilterResult.CONTINUE - } - if (document.body().html().length > maxLength) return@apply - // Unwrap single-child elements with no attributes - select("*").forEach { element -> - if (element.childNodes().size == 1 && element.childNodes()[0].nodeName() == "#text" && element.attributes().isEmpty()) { - element.unwrap() - } - } - if (document.body().html().length > maxLength) return@apply - // Convert relative URLs to absolute - select("[href],[src]").forEach { element -> - element.attr("href").let { href -> element.attr("href", href) } - element.attr("src").let { src -> element.attr("src", src) } - } - if (document.body().html().length > maxLength) return@apply - // Remove empty attributes - select("*").forEach { element -> - element.attributes().removeAll { it.value.isBlank() } - } - } - - // Truncate if necessary - val result = document.body().html() - return if (result.length > maxLength) { - result.substring(0, maxLength) - } else { - result - } - } - } -} private fun Node.text(): String { return this.childNodes().joinToString("") { it.text() } diff --git a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/file/TestGenerationTask.kt b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/file/TestGenerationTask.kt deleted file mode 100644 index c5ad2ae7..00000000 --- a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/file/TestGenerationTask.kt +++ /dev/null @@ -1,95 +0,0 @@ -package com.simiacryptus.skyenet.apps.plan.file - -import com.simiacryptus.jopenai.describe.Description -import com.simiacryptus.skyenet.apps.plan.PlanSettings -import com.simiacryptus.skyenet.apps.plan.TaskType -import com.simiacryptus.skyenet.apps.plan.file.TestGenerationTask.TestGenerationTaskConfigData -import org.slf4j.LoggerFactory - -class TestGenerationTask( - planSettings: PlanSettings, - planTask: TestGenerationTaskConfigData? -) : AbstractAnalysisTask(planSettings, planTask) { - - class TestGenerationTaskConfigData( - @Description("List of input files or tasks to be examined when generating tests") - val inputReferences: List? = null, - task_description: String? = null, - task_dependencies: List? = null, - input_files: List? = null, - output_files: List? = null, - state: TaskState? = null - ) : FileTaskConfigBase( - task_type = TaskType.TestGeneration.name, - task_description = task_description, - task_dependencies = task_dependencies, - input_files = input_files, - output_files = output_files, - state = state - ) - - override val actorName: String = "TestGeneration" - override val actorPrompt: String = """ - Generate comprehensive unit tests for the provided code files. The tests should: - |1. Cover all public methods and functions - |2. Include both positive and negative test cases - |3. Test edge cases and boundary conditions - |4. Ensure high code coverage - |5. Follow best practices for unit testing in the given programming language - | - |Provide the generated tests as complete, runnable code files. - |Use appropriate testing frameworks and assertion libraries for the target language. - |Include setup and teardown methods if necessary. - |Add comments explaining the purpose of each test case. - | - |Response format: - |- Use ${com.simiacryptus.skyenet.apps.plan.TRIPLE_TILDE} code blocks with a header specifying the new test file path. - |- Specify the language for syntax highlighting after the opening triple backticks. - |- Separate code blocks with a single blank line. - | - |Example: - | - |Here are the generated unit tests: - | - |### src/test/java/com/example/UtilsTest.java - |${com.simiacryptus.skyenet.apps.plan.TRIPLE_TILDE}java - |import org.junit.jupiter.api.Test; - |import static org.junit.jupiter.api.Assertions.*; - | - |public class UtilsTest { - | @Test - | public void testExampleFunction() { - | // Test the happy path - | assertEquals(3, Utils.exampleFunction(1, 2)); - | - | // Test edge cases - | assertEquals(0, Utils.exampleFunction(0, 0)); - | assertEquals(-1, Utils.exampleFunction(-1, 0)); - | - | // Test error conditions - | assertThrows(IllegalArgumentException.class, () -> { - | Utils.exampleFunction(Integer.MAX_VALUE, 1); - | }); - | } - |} - ${com.simiacryptus.skyenet.apps.plan.TRIPLE_TILDE} - """.trimMargin() - - override fun getAnalysisInstruction(): String = "Generate tests for the following code" - - override fun promptSegment(): String { - return """ - TestGeneration - Generate unit tests for the specified code files - ** Specify the files for which tests should be generated using the 'filesToTest' field - ** List input files/tasks to be examined when generating tests using the 'inputReferences' field - ** The task will generate test files for each specified file in 'filesToTest' - ** Test files will be created in a 'test' directory parallel to the source files - ** Specify the files for which tests should be generated - ** List input files/tasks to be examined when generating tests - """.trimMargin() - } - - companion object { - private val log = LoggerFactory.getLogger(TestGenerationTask::class.java) - } -} \ No newline at end of file diff --git a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/CommandAutoFixTask.kt b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/tools/CommandAutoFixTask.kt similarity index 97% rename from webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/CommandAutoFixTask.kt rename to webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/tools/CommandAutoFixTask.kt index 7c4793d7..d916089d 100644 --- a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/CommandAutoFixTask.kt +++ b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/tools/CommandAutoFixTask.kt @@ -1,4 +1,4 @@ -package com.simiacryptus.skyenet.apps.plan +package com.simiacryptus.skyenet.apps.plan.tools import com.simiacryptus.jopenai.ChatClient import com.simiacryptus.jopenai.OpenAIClient @@ -7,7 +7,7 @@ import com.simiacryptus.jopenai.models.ChatModel import com.simiacryptus.skyenet.Retryable import com.simiacryptus.skyenet.apps.general.CmdPatchApp import com.simiacryptus.skyenet.apps.general.PatchApp -import com.simiacryptus.skyenet.apps.plan.CommandAutoFixTask.CommandAutoFixTaskConfigData +import com.simiacryptus.skyenet.apps.plan.* import com.simiacryptus.skyenet.util.MarkdownUtil import com.simiacryptus.skyenet.webui.session.SessionTask import org.slf4j.LoggerFactory @@ -146,7 +146,7 @@ ${settings.commandAutoFixCommands?.joinToString("\n") { " * ${File(it).name}" ) { onComplete() } - if(autoRetries-- > 0) retryable?.retry() + if (autoRetries-- > 0) retryable?.retry() s }) task.placeholder diff --git a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/CommandSessionTask.kt b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/tools/CommandSessionTask.kt similarity index 89% rename from webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/CommandSessionTask.kt rename to webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/tools/CommandSessionTask.kt index 58bc4b1b..98e69566 100644 --- a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/CommandSessionTask.kt +++ b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/tools/CommandSessionTask.kt @@ -1,8 +1,9 @@ -package com.simiacryptus.skyenet.apps.plan +package com.simiacryptus.skyenet.apps.plan.tools import com.simiacryptus.jopenai.ChatClient import com.simiacryptus.jopenai.OpenAIClient import com.simiacryptus.jopenai.describe.Description +import com.simiacryptus.skyenet.apps.plan.* import com.simiacryptus.skyenet.webui.session.SessionTask import org.slf4j.LoggerFactory import java.io.BufferedReader @@ -12,8 +13,8 @@ import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.TimeUnit class CommandSessionTask( - planSettings: PlanSettings, - planTask: CommandSessionTaskConfigData? + planSettings: PlanSettings, + planTask: CommandSessionTaskConfigData? ) : AbstractTask(planSettings, planTask) { companion object { private val log = LoggerFactory.getLogger(CommandSessionTask::class.java) @@ -68,19 +69,19 @@ class CommandSessionTask( } class CommandSessionTaskConfigData( - @Description("The command to start the interactive session") + @Description("The command to start the interactive session") val command: List, - @Description("Commands to send to the interactive session") + @Description("Commands to send to the interactive session") val inputs: List = listOf(), - @Description("Session ID for reusing existing sessions") + @Description("Session ID for reusing existing sessions") val sessionId: String? = null, - @Description("Timeout in milliseconds for commands") + @Description("Timeout in milliseconds for commands") val timeout: Long = TIMEOUT_MS, - @Description("Whether to close the session after execution") + @Description("Whether to close the session after execution") val closeSession: Boolean = false, - task_description: String? = null, - task_dependencies: List? = null, - state: TaskState? = null, + task_description: String? = null, + task_dependencies: List? = null, + state: TaskState? = null, ) : TaskConfigBase( task_type = TaskType.CommandSession.name, task_description = task_description, @@ -105,13 +106,13 @@ class CommandSessionTask( } override fun run( - agent: PlanCoordinator, - messages: List, - task: SessionTask, - api: ChatClient, - resultFn: (String) -> Unit, - api2: OpenAIClient, - planSettings: PlanSettings + agent: PlanCoordinator, + messages: List, + task: SessionTask, + api: ChatClient, + resultFn: (String) -> Unit, + api2: OpenAIClient, + planSettings: PlanSettings ) { requireNotNull(taskConfig) { "CommandSessionTaskData is required" } var process: Process? = null diff --git a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/ForeachTask.kt b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/tools/ForeachTask.kt similarity index 71% rename from webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/ForeachTask.kt rename to webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/tools/ForeachTask.kt index 85289ce5..483316d9 100644 --- a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/ForeachTask.kt +++ b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/tools/ForeachTask.kt @@ -1,11 +1,9 @@ -package com.simiacryptus.skyenet.apps.plan +package com.simiacryptus.skyenet.apps.plan.tools import com.simiacryptus.jopenai.ChatClient import com.simiacryptus.jopenai.OpenAIClient import com.simiacryptus.jopenai.describe.Description -import com.simiacryptus.skyenet.apps.plan.ForeachTask.ForeachTaskConfigData -import com.simiacryptus.skyenet.apps.plan.PlanUtil.diagram -import com.simiacryptus.skyenet.apps.plan.PlanUtil.executionOrder +import com.simiacryptus.skyenet.apps.plan.* import com.simiacryptus.skyenet.webui.session.SessionTask import org.slf4j.LoggerFactory @@ -15,13 +13,13 @@ class ForeachTask( ) : AbstractTask(planSettings, planTask) { class ForeachTaskConfigData( - @Description("A list of items over which the ForEach task will iterate. (Only applicable for ForeachTask tasks) Can be used to process outputs from previous tasks.") + @Description("A list of items over which the ForEach task will iterate. (Only applicable for ForeachTask tasks) Can be used to process outputs from previous tasks.") val foreach_items: List? = null, - @Description("A map of sub-task IDs to PlanTask objects to be executed for each item. (Only applicable for ForeachTask tasks) Allows for complex task dependencies and information flow within iterations.") + @Description("A map of sub-task IDs to PlanTask objects to be executed for each item. (Only applicable for ForeachTask tasks) Allows for complex task dependencies and information flow within iterations.") val foreach_subplan: Map? = null, - task_description: String? = null, - task_dependencies: List? = null, - state: TaskState? = null, + task_description: String? = null, + task_dependencies: List? = null, + state: TaskState? = null, ) : TaskConfigBase( task_type = TaskType.ForeachTask.name, task_description = task_description, @@ -61,11 +59,11 @@ ForeachTask - Execute a task for each item in a list val itemPlanProcessingState = PlanProcessingState(itemSubTasks) agent.executePlan( task = subPlanTask, - diagramBuffer = subPlanTask.add(diagram(agent.ui, itemPlanProcessingState.subTasks)), + diagramBuffer = subPlanTask.add(PlanUtil.diagram(agent.ui, itemPlanProcessingState.subTasks)), subTasks = itemSubTasks, diagramTask = subPlanTask, planProcessingState = itemPlanProcessingState, - taskIdProcessingQueue = executionOrder(itemSubTasks).toMutableList(), + taskIdProcessingQueue = PlanUtil.executionOrder(itemSubTasks).toMutableList(), pool = agent.pool, userMessage = "$userMessage\nProcessing item $index: $item", plan = itemSubTasks, diff --git a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/GitHubSearchTask.kt b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/tools/GitHubSearchTask.kt similarity index 98% rename from webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/GitHubSearchTask.kt rename to webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/tools/GitHubSearchTask.kt index 3b3c3df6..29d853a2 100644 --- a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/GitHubSearchTask.kt +++ b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/tools/GitHubSearchTask.kt @@ -1,10 +1,11 @@ -package com.simiacryptus.skyenet.apps.plan +package com.simiacryptus.skyenet.apps.plan.tools import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.module.kotlin.readValue import com.simiacryptus.jopenai.ChatClient import com.simiacryptus.jopenai.OpenAIClient import com.simiacryptus.jopenai.describe.Description +import com.simiacryptus.skyenet.apps.plan.* import com.simiacryptus.skyenet.util.MarkdownUtil import com.simiacryptus.skyenet.webui.session.SessionTask import org.slf4j.LoggerFactory diff --git a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/GoogleSearchTask.kt b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/tools/GoogleSearchTask.kt similarity index 97% rename from webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/GoogleSearchTask.kt rename to webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/tools/GoogleSearchTask.kt index dc94aa12..23dfe8c5 100644 --- a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/GoogleSearchTask.kt +++ b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/tools/GoogleSearchTask.kt @@ -1,10 +1,11 @@ -package com.simiacryptus.skyenet.apps.plan +package com.simiacryptus.skyenet.apps.plan.tools import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.module.kotlin.readValue import com.simiacryptus.jopenai.ChatClient import com.simiacryptus.jopenai.OpenAIClient import com.simiacryptus.jopenai.describe.Description +import com.simiacryptus.skyenet.apps.plan.* import com.simiacryptus.skyenet.util.MarkdownUtil import com.simiacryptus.skyenet.webui.session.SessionTask import com.simiacryptus.util.JsonUtil diff --git a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/PlanningTask.kt b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/tools/PlanningTask.kt similarity index 85% rename from webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/PlanningTask.kt rename to webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/tools/PlanningTask.kt index 2cdbd273..dd158473 100644 --- a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/PlanningTask.kt +++ b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/tools/PlanningTask.kt @@ -1,18 +1,13 @@ -package com.simiacryptus.skyenet.apps.plan +package com.simiacryptus.skyenet.apps.plan.tools import com.simiacryptus.jopenai.API import com.simiacryptus.jopenai.ChatClient import com.simiacryptus.jopenai.OpenAIClient import com.simiacryptus.jopenai.describe.Description import com.simiacryptus.jopenai.models.ApiModel -import com.simiacryptus.jopenai.models.ApiModel.Role import com.simiacryptus.jopenai.util.ClientUtil.toContentList import com.simiacryptus.skyenet.Discussable -import com.simiacryptus.skyenet.apps.plan.PlanUtil.diagram -import com.simiacryptus.skyenet.apps.plan.PlanUtil.executionOrder -import com.simiacryptus.skyenet.apps.plan.PlanUtil.filterPlan -import com.simiacryptus.skyenet.apps.plan.PlanUtil.render -import com.simiacryptus.skyenet.apps.plan.PlanningTask.PlanningTaskConfigData +import com.simiacryptus.skyenet.apps.plan.* import com.simiacryptus.skyenet.core.actors.ParsedResponse import com.simiacryptus.skyenet.webui.application.ApplicationInterface import com.simiacryptus.skyenet.webui.session.SessionTask @@ -74,9 +69,9 @@ class PlanningTask( toInput("Expand ${taskConfig?.task_description ?: ""}"), api = api ) - render( + PlanUtil.render( withPrompt = TaskBreakdownWithPrompt( - plan = filterPlan { design.obj.tasksByID } ?: emptyMap(), + plan = PlanUtil.filterPlan { design.obj.tasksByID } ?: emptyMap(), planText = design.text, prompt = userMessage ), @@ -84,7 +79,7 @@ class PlanningTask( ) design.obj } - executeSubTasks(agent, userMessage, filterPlan { subPlan.tasksByID } ?: emptyMap(), task, api, api2) + executeSubTasks(agent, userMessage, PlanUtil.filterPlan { subPlan.tasksByID } ?: emptyMap(), task, api, api2) } private fun createSubPlanDiscussable( @@ -100,9 +95,9 @@ class PlanningTask( heading = "", initialResponse = { it: String -> planSettings.planningActor().answer(toInput(it), api = api) }, outputFn = { design: ParsedResponse -> - render( + PlanUtil.render( withPrompt = TaskBreakdownWithPrompt( - plan = filterPlan { design.obj.tasksByID } ?: emptyMap(), + plan = PlanUtil.filterPlan { design.obj.tasksByID } ?: emptyMap(), planText = design.text, prompt = userMessage ), @@ -110,7 +105,7 @@ class PlanningTask( ) }, ui = ui, - reviseResponse = { usermessages: List> -> + reviseResponse = { usermessages: List> -> planSettings.planningActor().respond( messages = usermessages.map { ApiModel.ChatMessage(it.second, it.first.toContentList()) } .toTypedArray(), @@ -139,11 +134,11 @@ class PlanningTask( ) ).executePlan( task = subPlanTask, - diagramBuffer = subPlanTask.add(diagram(coordinator.ui, planProcessingState.subTasks)), + diagramBuffer = subPlanTask.add(PlanUtil.diagram(coordinator.ui, planProcessingState.subTasks)), subTasks = subPlan, diagramTask = subPlanTask, planProcessingState = planProcessingState, - taskIdProcessingQueue = executionOrder(subPlan).toMutableList(), + taskIdProcessingQueue = PlanUtil.executionOrder(subPlan).toMutableList(), pool = coordinator.pool, userMessage = userMessage, plan = subPlan, diff --git a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/RunShellCommandTask.kt b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/tools/RunShellCommandTask.kt similarity index 95% rename from webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/RunShellCommandTask.kt rename to webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/tools/RunShellCommandTask.kt index 9d08fd9e..8a2dfd9c 100644 --- a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/RunShellCommandTask.kt +++ b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/tools/RunShellCommandTask.kt @@ -1,11 +1,11 @@ -package com.simiacryptus.skyenet.apps.plan +package com.simiacryptus.skyenet.apps.plan.tools import com.simiacryptus.jopenai.ChatClient import com.simiacryptus.jopenai.OpenAIClient import com.simiacryptus.jopenai.describe.Description import com.simiacryptus.jopenai.models.ApiModel import com.simiacryptus.skyenet.apps.code.CodingAgent -import com.simiacryptus.skyenet.apps.plan.RunShellCommandTask.RunShellCommandTaskConfigData +import com.simiacryptus.skyenet.apps.plan.* import com.simiacryptus.skyenet.core.actors.CodingActor import com.simiacryptus.skyenet.interpreter.ProcessInterpreter import com.simiacryptus.skyenet.webui.session.SessionTask @@ -121,9 +121,9 @@ Note: This task is for running simple and safe commands. Avoid executing command |${response.code} |$TRIPLE_TILDE | - |${TRIPLE_TILDE} + |$TRIPLE_TILDE |${response.renderedResponse} - |${TRIPLE_TILDE} + |$TRIPLE_TILDE | """.trimMargin() }.apply { resultFn(this) } diff --git a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/SearchTask.kt b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/tools/SearchTask.kt similarity index 97% rename from webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/SearchTask.kt rename to webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/tools/SearchTask.kt index 30430bdf..33f4008e 100644 --- a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/SearchTask.kt +++ b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/tools/SearchTask.kt @@ -1,9 +1,10 @@ -package com.simiacryptus.skyenet.apps.plan +package com.simiacryptus.skyenet.apps.plan.tools import com.simiacryptus.diff.FileValidationUtils import com.simiacryptus.jopenai.ChatClient import com.simiacryptus.jopenai.OpenAIClient import com.simiacryptus.jopenai.describe.Description +import com.simiacryptus.skyenet.apps.plan.* import com.simiacryptus.skyenet.util.MarkdownUtil import com.simiacryptus.skyenet.webui.session.SessionTask import org.slf4j.LoggerFactory @@ -18,7 +19,7 @@ class SearchTask( ) : AbstractTask(planSettings, planTask) { class SearchTaskConfigData( @Description("The search pattern (substring or regex) to look for in the files") - val search_pattern: String, + val search_pattern: String = "", @Description("Whether the search pattern is a regex (true) or a substring (false)") val is_regex: Boolean = false, @Description("The number of context lines to include before and after each match") diff --git a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/SeleniumSessionTask.kt b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/tools/SeleniumSessionTask.kt similarity index 96% rename from webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/SeleniumSessionTask.kt rename to webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/tools/SeleniumSessionTask.kt index edd1e7d3..5cab5e38 100644 --- a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/SeleniumSessionTask.kt +++ b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/tools/SeleniumSessionTask.kt @@ -1,11 +1,11 @@ -package com.simiacryptus.skyenet.apps.plan +package com.simiacryptus.skyenet.apps.plan.tools import com.simiacryptus.jopenai.ChatClient import com.simiacryptus.jopenai.OpenAIClient import com.simiacryptus.jopenai.describe.Description -import com.simiacryptus.skyenet.apps.plan.WebFetchAndTransformTask.Companion.scrubHtml +import com.simiacryptus.skyenet.apps.plan.* import com.simiacryptus.skyenet.core.util.Selenium -import com.simiacryptus.skyenet.util.MarkdownUtil.renderMarkdown +import com.simiacryptus.skyenet.util.MarkdownUtil import com.simiacryptus.skyenet.util.Selenium2S3 import com.simiacryptus.skyenet.webui.session.SessionTask import io.github.bonigarcia.wdm.WebDriverManager @@ -168,7 +168,7 @@ $activeSessionsInfo api2: OpenAIClient, planSettings: PlanSettings ) { - val seleniumFactory: (pool: java.util.concurrent.ThreadPoolExecutor, cookies: Array?) -> com.simiacryptus.skyenet.core.util.Selenium = + val seleniumFactory: (pool: java.util.concurrent.ThreadPoolExecutor, cookies: Array?) -> Selenium = { pool, cookies -> val chromeOptions = ChromeOptions().apply { addArguments("--headless") @@ -226,7 +226,7 @@ $activeSessionsInfo val result = formatResults(taskConfig, selenium, results) - task.add(renderMarkdown(result)) + task.add(MarkdownUtil.renderMarkdown(result)) resultFn(result) } finally { // Close session if it's temporary or explicitly requested to be closed diff --git a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/tools/WebFetchAndTransformTask.kt b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/tools/WebFetchAndTransformTask.kt new file mode 100644 index 00000000..4e767a53 --- /dev/null +++ b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/tools/WebFetchAndTransformTask.kt @@ -0,0 +1,147 @@ +package com.simiacryptus.skyenet.apps.plan.tools + +import com.simiacryptus.jopenai.API +import com.simiacryptus.jopenai.ChatClient +import com.simiacryptus.jopenai.OpenAIClient +import com.simiacryptus.jopenai.describe.Description +import com.simiacryptus.skyenet.apps.plan.* +import com.simiacryptus.skyenet.core.actors.SimpleActor +import com.simiacryptus.skyenet.util.MarkdownUtil +import com.simiacryptus.skyenet.webui.session.SessionTask +import org.apache.hc.client5.http.classic.methods.HttpGet +import org.apache.hc.client5.http.impl.classic.HttpClients +import org.apache.hc.core5.http.io.entity.EntityUtils +import org.jsoup.Jsoup +import org.jsoup.nodes.Document +import org.jsoup.select.NodeFilter +import org.slf4j.LoggerFactory + +open class WebFetchAndTransformTask( + planSettings: PlanSettings, + planTask: WebFetchAndTransformTaskConfigData? +) : AbstractTask(planSettings, planTask) { + class WebFetchAndTransformTaskConfigData( + @Description("The URL to fetch") + val url: String, + @Description("The desired format or focus for the transformation") + val transformationGoal: String, + task_description: String? = null, + task_dependencies: List? = null, + state: TaskState? = null, + ) : TaskConfigBase( + task_type = TaskType.WebFetchAndTransform.name, + task_description = task_description, + task_dependencies = task_dependencies, + state = state + ) + + override fun promptSegment() = """ + WebFetchAndTransform - Fetch a web page, strip HTML, and transform content + ** Specify the URL to fetch + ** Specify the desired format or focus for the transformation + """.trimMargin() + + override fun run( + agent: PlanCoordinator, + messages: List, + task: SessionTask, + api: ChatClient, + resultFn: (String) -> Unit, + api2: OpenAIClient, + planSettings: PlanSettings + ) { + val fetchedContent = fetchAndStripHtml(taskConfig?.url ?: "") + val transformedContent = transformContent(fetchedContent, taskConfig?.transformationGoal ?: "", api, planSettings) + task.add(MarkdownUtil.renderMarkdown(transformedContent, ui = agent.ui)) + resultFn(transformedContent) + } + + private fun fetchAndStripHtml(url: String): String { + HttpClients.createDefault().use { httpClient -> + val httpGet = HttpGet(url) + httpClient.execute(httpGet).use { response -> + val entity = response.entity + val content = EntityUtils.toString(entity) + return scrubHtml(content) + } + } + } + + private fun transformContent(content: String, transformationGoal: String, api: API, planSettings: PlanSettings): String { + val prompt = """ + Transform the following web content according to this goal: $transformationGoal + + Content: + $content + + Transformed content: + """.trimIndent() + return SimpleActor( + prompt = prompt, + model = planSettings.defaultModel, + ).answer( + listOf( + """ + |Transform the following web content according to this goal: $transformationGoal + | + |$content + """.trimMargin(), + ), api + ) + } + + companion object { + private val log = LoggerFactory.getLogger(WebFetchAndTransformTask::class.java) + fun scrubHtml(str: String, maxLength: Int = 100 * 1024): String { + val document: Document = Jsoup.parse(str) + // Remove unnecessary elements, attributes, and optimize the document + document.apply { + if (document.body().html().length > maxLength) return@apply + select("script, style, link, meta, iframe, noscript").remove() // Remove unnecessary and potentially harmful tags + outputSettings().prettyPrint(false) // Disable pretty printing for compact output + if (document.body().html().length > maxLength) return@apply + // Remove comments + select("*").forEach { it.childNodes().removeAll { node -> node.nodeName() == "#comment" } } + if (document.body().html().length > maxLength) return@apply + // Remove data-* attributes + select("*[data-*]").forEach { it.attributes().removeAll { attr -> attr.key.startsWith("data-") } } + if (document.body().html().length > maxLength) return@apply + select("*").forEach { element -> + val importantAttributes = setOf("href", "src", "alt", "title", "width", "height", "style", "class", "id", "name") + element.attributes().removeAll { it.key !in importantAttributes } + } + if (document.body().html().length > maxLength) return@apply + // Remove empty elements + select("*").filter { node, depth -> + if(node.text().isBlank() && node.attributes().isEmpty() && !node.hasAttr("img")) NodeFilter.FilterResult.REMOVE else NodeFilter.FilterResult.CONTINUE + } + if (document.body().html().length > maxLength) return@apply + // Unwrap single-child elements with no attributes + select("*").forEach { element -> + if (element.childNodes().size == 1 && element.childNodes()[0].nodeName() == "#text" && element.attributes().isEmpty()) { + element.unwrap() + } + } + if (document.body().html().length > maxLength) return@apply + // Convert relative URLs to absolute + select("[href],[src]").forEach { element -> + element.attr("href").let { href -> element.attr("href", href) } + element.attr("src").let { src -> element.attr("src", src) } + } + if (document.body().html().length > maxLength) return@apply + // Remove empty attributes + select("*").forEach { element -> + element.attributes().removeAll { it.value.isBlank() } + } + } + + // Truncate if necessary + val result = document.body().html() + return if (result.length > maxLength) { + result.substring(0, maxLength) + } else { + result + } + } + } +} \ No newline at end of file diff --git a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/file/AbstractAnalysisTask.kt b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/tools/file/AbstractAnalysisTask.kt similarity index 98% rename from webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/file/AbstractAnalysisTask.kt rename to webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/tools/file/AbstractAnalysisTask.kt index 4e9747ad..c300cde7 100644 --- a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/file/AbstractAnalysisTask.kt +++ b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/tools/file/AbstractAnalysisTask.kt @@ -1,4 +1,4 @@ -package com.simiacryptus.skyenet.apps.plan.file +package com.simiacryptus.skyenet.apps.plan.tools.file import com.simiacryptus.jopenai.API import com.simiacryptus.jopenai.ChatClient diff --git a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/file/AbstractFileTask.kt b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/tools/file/AbstractFileTask.kt similarity index 94% rename from webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/file/AbstractFileTask.kt rename to webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/tools/file/AbstractFileTask.kt index 625c017a..18b150e1 100644 --- a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/file/AbstractFileTask.kt +++ b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/tools/file/AbstractFileTask.kt @@ -1,11 +1,11 @@ -package com.simiacryptus.skyenet.apps.plan.file +package com.simiacryptus.skyenet.apps.plan.tools.file import com.simiacryptus.diff.FileValidationUtils import com.simiacryptus.jopenai.describe.Description import com.simiacryptus.skyenet.apps.plan.AbstractTask import com.simiacryptus.skyenet.apps.plan.PlanSettings import com.simiacryptus.skyenet.apps.plan.TaskConfigBase -import com.simiacryptus.skyenet.apps.plan.file.AbstractFileTask.FileTaskConfigBase +import com.simiacryptus.skyenet.apps.plan.tools.file.AbstractFileTask.FileTaskConfigBase import java.nio.file.FileSystems import java.nio.file.Files import kotlin.streams.asSequence diff --git a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/file/CodeOptimizationTask.kt b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/tools/file/CodeOptimizationTask.kt similarity index 93% rename from webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/file/CodeOptimizationTask.kt rename to webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/tools/file/CodeOptimizationTask.kt index 43b7fa6c..aef85d69 100644 --- a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/file/CodeOptimizationTask.kt +++ b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/tools/file/CodeOptimizationTask.kt @@ -1,9 +1,9 @@ -package com.simiacryptus.skyenet.apps.plan.file +package com.simiacryptus.skyenet.apps.plan.tools.file import com.simiacryptus.jopenai.describe.Description import com.simiacryptus.skyenet.apps.plan.PlanSettings import com.simiacryptus.skyenet.apps.plan.TaskType -import com.simiacryptus.skyenet.apps.plan.file.CodeOptimizationTask.CodeOptimizationTaskConfigData +import com.simiacryptus.skyenet.apps.plan.tools.file.CodeOptimizationTask.CodeOptimizationTaskConfigData import org.slf4j.LoggerFactory class CodeOptimizationTask( diff --git a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/file/CodeReviewTask.kt b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/tools/file/CodeReviewTask.kt similarity index 93% rename from webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/file/CodeReviewTask.kt rename to webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/tools/file/CodeReviewTask.kt index 600c71b4..466ca715 100644 --- a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/file/CodeReviewTask.kt +++ b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/tools/file/CodeReviewTask.kt @@ -1,9 +1,9 @@ -package com.simiacryptus.skyenet.apps.plan.file +package com.simiacryptus.skyenet.apps.plan.tools.file import com.simiacryptus.jopenai.describe.Description import com.simiacryptus.skyenet.apps.plan.PlanSettings import com.simiacryptus.skyenet.apps.plan.TaskType -import com.simiacryptus.skyenet.apps.plan.file.CodeReviewTask.CodeReviewTaskConfigData +import com.simiacryptus.skyenet.apps.plan.tools.file.CodeReviewTask.CodeReviewTaskConfigData import org.slf4j.LoggerFactory class CodeReviewTask( diff --git a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/file/DocumentationTask.kt b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/tools/file/DocumentationTask.kt similarity index 97% rename from webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/file/DocumentationTask.kt rename to webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/tools/file/DocumentationTask.kt index 2cdd0de5..7b78e034 100644 --- a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/file/DocumentationTask.kt +++ b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/tools/file/DocumentationTask.kt @@ -1,4 +1,4 @@ -package com.simiacryptus.skyenet.apps.plan.file +package com.simiacryptus.skyenet.apps.plan.tools.file import com.simiacryptus.diff.addApplyFileDiffLinks import com.simiacryptus.jopenai.ChatClient @@ -8,7 +8,7 @@ import com.simiacryptus.skyenet.Retryable import com.simiacryptus.skyenet.apps.plan.PlanCoordinator import com.simiacryptus.skyenet.apps.plan.PlanSettings import com.simiacryptus.skyenet.apps.plan.TaskType -import com.simiacryptus.skyenet.apps.plan.file.DocumentationTask.DocumentationTaskConfigData +import com.simiacryptus.skyenet.apps.plan.tools.file.DocumentationTask.DocumentationTaskConfigData import com.simiacryptus.skyenet.core.actors.SimpleActor import com.simiacryptus.skyenet.util.MarkdownUtil import com.simiacryptus.skyenet.webui.session.SessionTask diff --git a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/file/FileModificationTask.kt b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/tools/file/FileModificationTask.kt similarity index 97% rename from webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/file/FileModificationTask.kt rename to webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/tools/file/FileModificationTask.kt index 86e402ff..937aeebf 100644 --- a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/file/FileModificationTask.kt +++ b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/tools/file/FileModificationTask.kt @@ -1,4 +1,4 @@ -package com.simiacryptus.skyenet.apps.plan.file +package com.simiacryptus.skyenet.apps.plan.tools.file import com.simiacryptus.diff.addApplyFileDiffLinks import com.simiacryptus.jopenai.ChatClient @@ -8,7 +8,7 @@ import com.simiacryptus.skyenet.Retryable import com.simiacryptus.skyenet.apps.plan.PlanCoordinator import com.simiacryptus.skyenet.apps.plan.PlanSettings import com.simiacryptus.skyenet.apps.plan.TaskType -import com.simiacryptus.skyenet.apps.plan.file.FileModificationTask.FileModificationTaskConfigData +import com.simiacryptus.skyenet.apps.plan.tools.file.FileModificationTask.FileModificationTaskConfigData import com.simiacryptus.skyenet.core.actors.SimpleActor import com.simiacryptus.skyenet.util.MarkdownUtil.renderMarkdown import com.simiacryptus.skyenet.webui.session.SessionTask diff --git a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/file/InquiryTask.kt b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/tools/file/InquiryTask.kt similarity index 99% rename from webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/file/InquiryTask.kt rename to webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/tools/file/InquiryTask.kt index 22e83a10..a274472d 100644 --- a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/file/InquiryTask.kt +++ b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/tools/file/InquiryTask.kt @@ -1,4 +1,4 @@ -package com.simiacryptus.skyenet.apps.plan.file +package com.simiacryptus.skyenet.apps.plan.tools.file import com.simiacryptus.diff.FileValidationUtils import com.simiacryptus.jopenai.ChatClient diff --git a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/file/PerformanceAnalysisTask.kt b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/tools/file/PerformanceAnalysisTask.kt similarity index 93% rename from webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/file/PerformanceAnalysisTask.kt rename to webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/tools/file/PerformanceAnalysisTask.kt index 59cfee89..804ca928 100644 --- a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/file/PerformanceAnalysisTask.kt +++ b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/tools/file/PerformanceAnalysisTask.kt @@ -1,9 +1,9 @@ -package com.simiacryptus.skyenet.apps.plan.file +package com.simiacryptus.skyenet.apps.plan.tools.file import com.simiacryptus.jopenai.describe.Description import com.simiacryptus.skyenet.apps.plan.PlanSettings import com.simiacryptus.skyenet.apps.plan.TaskType -import com.simiacryptus.skyenet.apps.plan.file.PerformanceAnalysisTask.PerformanceAnalysisTaskConfigData +import com.simiacryptus.skyenet.apps.plan.tools.file.PerformanceAnalysisTask.PerformanceAnalysisTaskConfigData import org.slf4j.LoggerFactory class PerformanceAnalysisTask( diff --git a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/file/RefactorTask.kt b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/tools/file/RefactorTask.kt similarity index 94% rename from webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/file/RefactorTask.kt rename to webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/tools/file/RefactorTask.kt index 977b2ba8..c6fea477 100644 --- a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/file/RefactorTask.kt +++ b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/tools/file/RefactorTask.kt @@ -1,9 +1,9 @@ -package com.simiacryptus.skyenet.apps.plan.file +package com.simiacryptus.skyenet.apps.plan.tools.file import com.simiacryptus.jopenai.describe.Description import com.simiacryptus.skyenet.apps.plan.PlanSettings import com.simiacryptus.skyenet.apps.plan.TaskType -import com.simiacryptus.skyenet.apps.plan.file.RefactorTask.RefactorTaskConfigData +import com.simiacryptus.skyenet.apps.plan.tools.file.RefactorTask.RefactorTaskConfigData import org.slf4j.LoggerFactory class RefactorTask( diff --git a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/file/SecurityAuditTask.kt b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/tools/file/SecurityAuditTask.kt similarity index 93% rename from webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/file/SecurityAuditTask.kt rename to webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/tools/file/SecurityAuditTask.kt index c0de5ba9..af64d21c 100644 --- a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/file/SecurityAuditTask.kt +++ b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/tools/file/SecurityAuditTask.kt @@ -1,9 +1,9 @@ -package com.simiacryptus.skyenet.apps.plan.file +package com.simiacryptus.skyenet.apps.plan.tools.file import com.simiacryptus.jopenai.describe.Description import com.simiacryptus.skyenet.apps.plan.PlanSettings import com.simiacryptus.skyenet.apps.plan.TaskType -import com.simiacryptus.skyenet.apps.plan.file.SecurityAuditTask.SecurityAuditTaskConfigData +import com.simiacryptus.skyenet.apps.plan.tools.file.SecurityAuditTask.SecurityAuditTaskConfigData import org.slf4j.LoggerFactory class SecurityAuditTask( diff --git a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/tools/file/TestGenerationTask.kt b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/tools/file/TestGenerationTask.kt new file mode 100644 index 00000000..ba49235d --- /dev/null +++ b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/tools/file/TestGenerationTask.kt @@ -0,0 +1,94 @@ +package com.simiacryptus.skyenet.apps.plan.tools.file + +import com.simiacryptus.jopenai.describe.Description +import com.simiacryptus.skyenet.apps.plan.PlanSettings +import com.simiacryptus.skyenet.apps.plan.TRIPLE_TILDE +import com.simiacryptus.skyenet.apps.plan.TaskType +import com.simiacryptus.skyenet.apps.plan.tools.file.TestGenerationTask.TestGenerationTaskConfigData +import org.slf4j.LoggerFactory + +class TestGenerationTask( + planSettings: PlanSettings, + planTask: TestGenerationTaskConfigData? +) : AbstractAnalysisTask(planSettings, planTask) { + + class TestGenerationTaskConfigData( + task_description: String? = null, + task_dependencies: List? = null, + input_files: List? = null, + output_files: List? = null, + state: TaskState? = null + ) : FileTaskConfigBase( + task_type = TaskType.TestGeneration.name, + task_description = task_description, + task_dependencies = task_dependencies, + input_files = input_files, + output_files = output_files, + state = state + ) + + override val actorName: String = "TestGeneration" + override val actorPrompt: String = """ +Generate comprehensive unit tests for the provided code files. The tests should: +1. Cover all public methods and functions +2. Include both positive and negative test cases +3. Test edge cases and boundary conditions +4. Ensure high code coverage +5. Follow best practices for unit testing in the given programming language + +Provide the generated tests as complete, runnable code files. +Use appropriate testing frameworks and assertion libraries for the target language. +Include setup and teardown methods if necessary. +Add comments explaining the purpose of each test case. + +Response format: +- Use ${com.simiacryptus.skyenet.apps.plan.TRIPLE_TILDE} code blocks with a header specifying the new test file path. +- Specify the language for syntax highlighting after the opening triple backticks. +- Separate code blocks with a single blank line. + +Example: + +Here are the generated unit tests: + +### src/test/java/com/example/UtilsTest.java +${com.simiacryptus.skyenet.apps.plan.TRIPLE_TILDE}java +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.*; + +public class UtilsTest { + @Test + public void testExampleFunction() { + // Test the happy path + assertEquals(3, Utils.exampleFunction(1, 2)); + + // Test edge cases + assertEquals(0, Utils.exampleFunction(0, 0)); + assertEquals(-1, Utils.exampleFunction(-1, 0)); + + // Test error conditions + assertThrows(IllegalArgumentException.class, () -> { + Utils.exampleFunction(Integer.MAX_VALUE, 1); + }); + } +} +${com.simiacryptus.skyenet.apps.plan.TRIPLE_TILDE} +""".trimIndent() + + override fun getAnalysisInstruction(): String = "Generate tests for the following code" + + override fun promptSegment(): String { + return """ +TestGeneration - Generate unit tests for the specified code files +* Specify the files for which tests should be generated using the 'filesToTest' field +* List input files/tasks to be examined when generating tests using the 'inputReferences' field +* The task will generate test files for each specified file in 'filesToTest' +* Test files will be created in a 'test' directory parallel to the source files + * Specify the files for which tests should be generated + * List input files/tasks to be examined when generating tests + """.trimIndent() + } + + companion object { + private val log = LoggerFactory.getLogger(TestGenerationTask::class.java) + } +} \ No newline at end of file diff --git a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/knowledge/EmbeddingSearchTask.kt b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/tools/knowledge/EmbeddingSearchTask.kt similarity index 99% rename from webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/knowledge/EmbeddingSearchTask.kt rename to webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/tools/knowledge/EmbeddingSearchTask.kt index d861c666..0249913c 100644 --- a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/knowledge/EmbeddingSearchTask.kt +++ b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/tools/knowledge/EmbeddingSearchTask.kt @@ -1,4 +1,4 @@ -package com.simiacryptus.skyenet.apps.plan.knowledge +package com.simiacryptus.skyenet.apps.plan.tools.knowledge import com.fasterxml.jackson.databind.JsonNode import com.fasterxml.jackson.databind.ObjectMapper diff --git a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/knowledge/KnowledgeIndexingTask.kt b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/tools/knowledge/KnowledgeIndexingTask.kt similarity index 98% rename from webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/knowledge/KnowledgeIndexingTask.kt rename to webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/tools/knowledge/KnowledgeIndexingTask.kt index c7a92667..232a191e 100644 --- a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/knowledge/KnowledgeIndexingTask.kt +++ b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/tools/knowledge/KnowledgeIndexingTask.kt @@ -1,4 +1,4 @@ -package com.simiacryptus.skyenet.apps.plan.knowledge +package com.simiacryptus.skyenet.apps.plan.tools.knowledge import com.simiacryptus.jopenai.ChatClient import com.simiacryptus.jopenai.OpenAIClient diff --git a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/knowledge/WebSearchAndIndexTask.kt b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/tools/knowledge/WebSearchAndIndexTask.kt similarity index 98% rename from webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/knowledge/WebSearchAndIndexTask.kt rename to webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/tools/knowledge/WebSearchAndIndexTask.kt index 39d71a32..95d99e2a 100644 --- a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/knowledge/WebSearchAndIndexTask.kt +++ b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/tools/knowledge/WebSearchAndIndexTask.kt @@ -1,4 +1,4 @@ -package com.simiacryptus.skyenet.apps.plan.knowledge +package com.simiacryptus.skyenet.apps.plan.tools.knowledge import com.simiacryptus.jopenai.ChatClient import com.simiacryptus.jopenai.OpenAIClient