1.2.18 (#119)

gradle.properties

@@ -1,5 +1,5 @@
 # Gradle Releases -> https://github.com/gradle/gradle/releases
 libraryGroup=com.simiacryptus.skyenet
-libraryVersion=1.2.17
+libraryVersion=1.2.18
 gradleVersion=7.6.1
 kotlin.daemon.jvmargs=-Xmx4g

webui/src/main/kotlin/com/simiacryptus/skyenet/Retryable.kt

@@ -19,25 +19,24 @@ open class Retryable(
     set(tabLabel, process(container))
   }
 
+  fun retry() {
+    val idx = tabs.size
+    val label = label(idx)
+    val content = StringBuilder("Retrying..." + SessionTask.spinner)
+    tabs.add(label to content)
+    update()
+    val newResult = process(content)
+    content.clear()
+    set(label, newResult)
+  }
+
   override fun renderTabButtons(): String = """
 <div class="tabs">${
     tabs.withIndex().joinToString("\n") { (index, _) ->
       val tabId = "$index"
       """<button class="tab-button" data-for-tab="$tabId">${index + 1}</button>"""
     }
-  }
-${
-    ui.hrefLink("♻") {
-      val idx = tabs.size
-      val label = label(idx)
-      val content = StringBuilder("Retrying..." + SessionTask.spinner)
-      tabs.add(label to content)
-      update()
-      val newResult = process(content)
-      content.clear()
-      set(label, newResult)
-    }
-  }
+  }${ui.hrefLink("♻") { retry() }}
 </div>
 """
 

webui/src/main/kotlin/com/simiacryptus/skyenet/TabbedDisplay.kt

@@ -14,15 +14,18 @@ open class TabbedDisplay(
   }
 
   val size: Int get() = tabs.size
-  open fun render() = if (tabs.isEmpty()) "<div/>" else """
-<div class="tabs-container" id="${UUID.randomUUID()}">
-${renderTabButtons()}
-${
-    tabs.toTypedArray().withIndex().joinToString("\n")
-    { (idx, t) -> renderContentTab(t, idx) }
+  val tabId = UUID.randomUUID()
+  open fun render() = if (tabs.isEmpty()) "<div/>" else {
+    """
+  <div class="tabs-container" id="$tabId">
+  ${renderTabButtons()}
+  ${
+      tabs.toTypedArray().withIndex().joinToString("\n")
+      { (idx, t) -> renderContentTab(t, idx) }
+    }
+  </div>
+  """
   }
-</div>
-"""
 
   val container: StringBuilder by lazy {
     log.debug("Initializing container with rendered content")

webui/src/main/kotlin/com/simiacryptus/skyenet/apps/parse/DocumentParserApp.kt

@@ -31,6 +31,8 @@ open class DocumentParserApp(
   val reader: (File) -> DocumentReader = {
     when {
       it.name.endsWith(".pdf", ignoreCase = true) -> PDFReader(it)
+      it.name.endsWith(".html", ignoreCase = true) -> HTMLReader(it)
+      it.name.endsWith(".htm", ignoreCase = true) -> HTMLReader(it)
       else -> TextReader(it)
     }
   },

webui/src/main/kotlin/com/simiacryptus/skyenet/apps/parse/HTMLReader.kt

@@ -0,0 +1,82 @@
+package com.simiacryptus.skyenet.apps.parse
+
+import org.jsoup.Jsoup
+import org.jsoup.nodes.Document
+import java.awt.image.BufferedImage
+import java.io.File
+
+class HTMLReader(private val htmlFile: File) : DocumentParserApp.DocumentReader {
+    private val document: Document = Jsoup.parse(htmlFile, "UTF-8")
+    private val pages: List<String> = splitIntoPages(document.body().text())
+    private lateinit var settings: DocumentParserApp.Settings
+
+    fun configure(settings: DocumentParserApp.Settings) {
+        this.settings = settings
+    }
+
+    override fun getPageCount(): Int = pages.size
+
+    override fun getText(startPage: Int, endPage: Int): String {
+        val text = pages.subList(startPage, endPage.coerceAtMost(pages.size)).joinToString("\n")
+        return if (::settings.isInitialized && settings.addLineNumbers) {
+            text.lines().mapIndexed { index, line -> 
+                "${(index + 1).toString().padStart(6)}: $line"
+            }.joinToString("\n")
+        } else text
+    }
+
+    override fun renderImage(pageIndex: Int, dpi: Float): BufferedImage {
+        throw UnsupportedOperationException("HTML files do not support image rendering")
+    }
+
+    override fun close() {
+        // No resources to close for HTML files
+    }
+
+    private fun splitIntoPages(text: String, maxChars: Int = 16000): List<String> {
+        if (text.length <= maxChars) return listOf(text)
+
+        // Split on paragraph boundaries when possible
+        val paragraphs = text.split(Regex("\\n\\s*\\n"))
+
+        val pages = mutableListOf<String>()
+        var currentPage = StringBuilder()
+
+        for (paragraph in paragraphs) {
+            if (currentPage.length + paragraph.length > maxChars) {
+                if (currentPage.isNotEmpty()) {
+                    pages.add(currentPage.toString())
+                    currentPage = StringBuilder()
+                }
+                // If a single paragraph is longer than maxChars, split it
+                if (paragraph.length > maxChars) {
+                    val words = paragraph.split(" ")
+                    var currentChunk = StringBuilder()
+
+                    for (word in words) {
+                        if (currentChunk.length + word.length > maxChars) {
+                            pages.add(currentChunk.toString())
+                            currentChunk = StringBuilder()
+                        }
+                        if (currentChunk.isNotEmpty()) currentChunk.append(" ")
+                        currentChunk.append(word)
+                    }
+                    if (currentChunk.isNotEmpty()) {
+                        currentPage.append(currentChunk)
+                    }
+                } else {
+                    currentPage.append(paragraph)
+                }
+            } else {
+                if (currentPage.isNotEmpty()) currentPage.append("\n\n")
+                currentPage.append(paragraph)
+            }
+        }
+
+        if (currentPage.isNotEmpty()) {
+            pages.add(currentPage.toString())
+        }
+
+        return pages
+    }
+}

webui/src/main/kotlin/com/simiacryptus/skyenet/apps/parse/LogDataParsingModel.kt

@@ -34,27 +34,55 @@ open class LogDataParsingModel(
 
     override fun getFastParser(api: API): (String) -> LogData {
         val patternGenerator = LogPatternGenerator(parsingModel, temperature)
-
         return { originalText ->
-            var remainingText = originalText
-            var result: LogData? = null
-            var iterationCount = 0
+            parseText(originalText, patternGenerator, api, emptyList())
+        }
+    }
 
-            try {
-                while (remainingText.isNotBlank() && iterationCount++ < maxIterations) {
-                    val patterns = patternGenerator.generatePatterns(api, remainingText)
-                    if (patterns.isEmpty()) break
-                    val applyPatterns = applyPatterns(remainingText, (result?.patterns ?: emptyList()) + patterns)
-                    result = applyPatterns.first
-                    remainingText = applyPatterns.second
+    override fun getSmartParser(api: API): (LogData, String) -> LogData {
+        val patternGenerator = LogPatternGenerator(parsingModel, temperature)
+        return { runningDocument, prompt ->
+            parseText(prompt, patternGenerator, api, runningDocument.patterns ?: emptyList())
+        }
+    }
+
+    private fun parseText(
+        originalText: String,
+        patternGenerator: LogPatternGenerator,
+        api: API,
+        existingPatterns: List<PatternData>
+    ): LogData {
+        var remainingText = originalText
+        var result: LogData? = null
+        var iterationCount = 0
+        var currentPatterns = existingPatterns
+        try {
+            // First try with existing patterns
+            if (currentPatterns.isNotEmpty()) {
+                val applyPatterns = applyPatterns(remainingText, currentPatterns)
+                result = applyPatterns.first
+                remainingText = applyPatterns.second
+            }
+            // Then generate new patterns for remaining text
+            while (remainingText.isNotBlank() && iterationCount++ < maxIterations) {
+                val newPatterns = patternGenerator.generatePatterns(api, remainingText)
+                if (newPatterns.isEmpty()) break
+                currentPatterns = (currentPatterns + newPatterns).distinctBy { it.regex }
+                val applyPatterns = applyPatterns(remainingText, currentPatterns)
+                result = if (result != null) {
+                    merge(result, applyPatterns.first)
+                } else {
+                    applyPatterns.first
                 }
-            } catch (e: Exception) {
-                log.error("Error parsing log data", e)
+                remainingText = applyPatterns.second
             }
-            result ?: LogData()
+        } catch (e: Exception) {
+            log.error("Error parsing log data", e)
         }
+        return result ?: LogData()
     }
 
+
     private fun applyPatterns(text: String, patterns: List<PatternData>): Pair<LogData, String> {
         val patterns = patterns.filter { it.regex != null }.groupBy { it.id }.map { it.value.first() }
         val matches = patterns.flatMap { pattern ->

webui/src/main/kotlin/com/simiacryptus/skyenet/apps/parse/TextReader.kt

@@ -41,10 +41,10 @@ class TextReader(private val textFile: File) : DocumentParserApp.DocumentReader
       var fitness = -((leftSize.toDouble() / text.length) * Math.log1p(rightSize.toDouble() / text.length) +
           (rightSize.toDouble() / text.length) * Math.log1p(leftSize.toDouble() / text.length))
       if (lines[i].isEmpty()) fitness *= 2
-      i to fitness.toDouble()
+      i to fitness
     }.toTypedArray().toMutableList()
 
-    var bestSplitIndex = splitFitnesses.minByOrNull { it.second }?.first ?: lines.size / 2
+    val bestSplitIndex = splitFitnesses.minByOrNull { it.second }?.first ?: lines.size / 2
     val leftText = lines.subList(0, bestSplitIndex).joinToString("\n")
     val rightText = lines.subList(bestSplitIndex, lines.size).joinToString("\n")
     return splitIntoPages(leftText, maxChars) + splitIntoPages(rightText, maxChars)

webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/CommandAutoFixTask.kt

@@ -60,10 +60,12 @@ ${planSettings.commandAutoFixCommands?.joinToString("\n") { "    * ${File(it).na
     api2: OpenAIClient,
     planSettings: PlanSettings
   ) {
+    var autoRetries = if(planSettings.autoFix) 5 else 0
     val semaphore = Semaphore(0)
     val hasError = AtomicBoolean(false)
     val onComplete = { semaphore.release() }
-    Retryable(agent.ui, task = task) {
+    lateinit var retryable: Retryable
+    retryable = Retryable(agent.ui, task = task) {
       val task = agent.ui.newTask(false).apply { it.append(placeholder) }
       this.planTask?.commands?.forEachIndexed { index, commandWithDir ->
         val alias = commandWithDir.command.firstOrNull()
@@ -122,19 +124,22 @@ ${planSettings.commandAutoFixCommands?.joinToString("\n") { "    * ${File(it).na
         )
       }
       resultFn("All Command Auto Fix tasks completed")
-      task.add(if (!hasError.get()) {
-        onComplete()
-        MarkdownUtil.renderMarkdown("## All Command Auto Fix tasks completed successfully\n", ui = agent.ui, tabs = false)
-      } else {
-        MarkdownUtil.renderMarkdown(
-          "## Some Command Auto Fix tasks failed\n",
-          ui = agent.ui
-        ) + acceptButtonFooter(
-          agent.ui
-        ) {
+      task.add(
+        if (!hasError.get()) {
           onComplete()
-        }
-      })
+          MarkdownUtil.renderMarkdown("## All Command Auto Fix tasks completed successfully\n", ui = agent.ui, tabs = false)
+        } else {
+          val s = MarkdownUtil.renderMarkdown(
+            "## Some Command Auto Fix tasks failed\n",
+            ui = agent.ui
+          ) + acceptButtonFooter(
+            agent.ui
+          ) {
+            onComplete()
+          }
+          if(autoRetries-- > 0) retryable.retry()
+          s
+        })
       task.placeholder
     }
     try {