From f8467e1412c9a01df1d71c1fa1845e3e66ff8c01 Mon Sep 17 00:00:00 2001 From: Andrew Charneski Date: Sat, 16 Nov 2024 13:53:33 -0500 Subject: [PATCH] 1.2.18 (#119) --- gradle.properties | 2 +- .../com/simiacryptus/skyenet/Retryable.kt | 25 ++- .../com/simiacryptus/skyenet/TabbedDisplay.kt | 19 +- .../skyenet/apps/parse/DocumentParserApp.kt | 2 + .../skyenet/apps/parse/HTMLReader.kt | 82 +++++++ .../skyenet/apps/parse/LogDataParsingModel.kt | 56 +++-- .../skyenet/apps/parse/TextReader.kt | 4 +- .../skyenet/apps/plan/CommandAutoFixTask.kt | 31 +-- .../src/main/resources/application/README.md | 201 ++++++++++++++++++ webui/src/main/resources/application/main.js | 17 -- webui/src/main/resources/application/tabs.js | 47 ++-- 11 files changed, 399 insertions(+), 87 deletions(-) create mode 100644 webui/src/main/kotlin/com/simiacryptus/skyenet/apps/parse/HTMLReader.kt create mode 100644 webui/src/main/resources/application/README.md diff --git a/gradle.properties b/gradle.properties index 4e9c579b..4d3feb2e 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,5 +1,5 @@ # Gradle Releases -> https://github.com/gradle/gradle/releases libraryGroup=com.simiacryptus.skyenet -libraryVersion=1.2.17 +libraryVersion=1.2.18 gradleVersion=7.6.1 kotlin.daemon.jvmargs=-Xmx4g diff --git a/webui/src/main/kotlin/com/simiacryptus/skyenet/Retryable.kt b/webui/src/main/kotlin/com/simiacryptus/skyenet/Retryable.kt index 77376467..9daff02b 100644 --- a/webui/src/main/kotlin/com/simiacryptus/skyenet/Retryable.kt +++ b/webui/src/main/kotlin/com/simiacryptus/skyenet/Retryable.kt @@ -19,25 +19,24 @@ open class Retryable( set(tabLabel, process(container)) } + fun retry() { + val idx = tabs.size + val label = label(idx) + val content = StringBuilder("Retrying..." + SessionTask.spinner) + tabs.add(label to content) + update() + val newResult = process(content) + content.clear() + set(label, newResult) + } + override fun renderTabButtons(): String = """
${ tabs.withIndex().joinToString("\n") { (index, _) -> val tabId = "$index" """""" } - } -${ - ui.hrefLink("♻") { - val idx = tabs.size - val label = label(idx) - val content = StringBuilder("Retrying..." + SessionTask.spinner) - tabs.add(label to content) - update() - val newResult = process(content) - content.clear() - set(label, newResult) - } - } + }${ui.hrefLink("♻") { retry() }}
""" diff --git a/webui/src/main/kotlin/com/simiacryptus/skyenet/TabbedDisplay.kt b/webui/src/main/kotlin/com/simiacryptus/skyenet/TabbedDisplay.kt index c4069767..719d1733 100644 --- a/webui/src/main/kotlin/com/simiacryptus/skyenet/TabbedDisplay.kt +++ b/webui/src/main/kotlin/com/simiacryptus/skyenet/TabbedDisplay.kt @@ -14,15 +14,18 @@ open class TabbedDisplay( } val size: Int get() = tabs.size - open fun render() = if (tabs.isEmpty()) "
" else """ -
-${renderTabButtons()} -${ - tabs.toTypedArray().withIndex().joinToString("\n") - { (idx, t) -> renderContentTab(t, idx) } + val tabId = UUID.randomUUID() + open fun render() = if (tabs.isEmpty()) "
" else { + """ +
+ ${renderTabButtons()} + ${ + tabs.toTypedArray().withIndex().joinToString("\n") + { (idx, t) -> renderContentTab(t, idx) } + } +
+ """ } -
-""" val container: StringBuilder by lazy { log.debug("Initializing container with rendered content") diff --git a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/parse/DocumentParserApp.kt b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/parse/DocumentParserApp.kt index a8ee26d1..ad2ed28a 100644 --- a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/parse/DocumentParserApp.kt +++ b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/parse/DocumentParserApp.kt @@ -31,6 +31,8 @@ open class DocumentParserApp( val reader: (File) -> DocumentReader = { when { it.name.endsWith(".pdf", ignoreCase = true) -> PDFReader(it) + it.name.endsWith(".html", ignoreCase = true) -> HTMLReader(it) + it.name.endsWith(".htm", ignoreCase = true) -> HTMLReader(it) else -> TextReader(it) } }, diff --git a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/parse/HTMLReader.kt b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/parse/HTMLReader.kt new file mode 100644 index 00000000..72182d14 --- /dev/null +++ b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/parse/HTMLReader.kt @@ -0,0 +1,82 @@ +package com.simiacryptus.skyenet.apps.parse + +import org.jsoup.Jsoup +import org.jsoup.nodes.Document +import java.awt.image.BufferedImage +import java.io.File + +class HTMLReader(private val htmlFile: File) : DocumentParserApp.DocumentReader { + private val document: Document = Jsoup.parse(htmlFile, "UTF-8") + private val pages: List = splitIntoPages(document.body().text()) + private lateinit var settings: DocumentParserApp.Settings + + fun configure(settings: DocumentParserApp.Settings) { + this.settings = settings + } + + override fun getPageCount(): Int = pages.size + + override fun getText(startPage: Int, endPage: Int): String { + val text = pages.subList(startPage, endPage.coerceAtMost(pages.size)).joinToString("\n") + return if (::settings.isInitialized && settings.addLineNumbers) { + text.lines().mapIndexed { index, line -> + "${(index + 1).toString().padStart(6)}: $line" + }.joinToString("\n") + } else text + } + + override fun renderImage(pageIndex: Int, dpi: Float): BufferedImage { + throw UnsupportedOperationException("HTML files do not support image rendering") + } + + override fun close() { + // No resources to close for HTML files + } + + private fun splitIntoPages(text: String, maxChars: Int = 16000): List { + if (text.length <= maxChars) return listOf(text) + + // Split on paragraph boundaries when possible + val paragraphs = text.split(Regex("\\n\\s*\\n")) + + val pages = mutableListOf() + var currentPage = StringBuilder() + + for (paragraph in paragraphs) { + if (currentPage.length + paragraph.length > maxChars) { + if (currentPage.isNotEmpty()) { + pages.add(currentPage.toString()) + currentPage = StringBuilder() + } + // If a single paragraph is longer than maxChars, split it + if (paragraph.length > maxChars) { + val words = paragraph.split(" ") + var currentChunk = StringBuilder() + + for (word in words) { + if (currentChunk.length + word.length > maxChars) { + pages.add(currentChunk.toString()) + currentChunk = StringBuilder() + } + if (currentChunk.isNotEmpty()) currentChunk.append(" ") + currentChunk.append(word) + } + if (currentChunk.isNotEmpty()) { + currentPage.append(currentChunk) + } + } else { + currentPage.append(paragraph) + } + } else { + if (currentPage.isNotEmpty()) currentPage.append("\n\n") + currentPage.append(paragraph) + } + } + + if (currentPage.isNotEmpty()) { + pages.add(currentPage.toString()) + } + + return pages + } +} \ No newline at end of file diff --git a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/parse/LogDataParsingModel.kt b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/parse/LogDataParsingModel.kt index 23850f49..38fa7430 100644 --- a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/parse/LogDataParsingModel.kt +++ b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/parse/LogDataParsingModel.kt @@ -34,27 +34,55 @@ open class LogDataParsingModel( override fun getFastParser(api: API): (String) -> LogData { val patternGenerator = LogPatternGenerator(parsingModel, temperature) - return { originalText -> - var remainingText = originalText - var result: LogData? = null - var iterationCount = 0 + parseText(originalText, patternGenerator, api, emptyList()) + } + } - try { - while (remainingText.isNotBlank() && iterationCount++ < maxIterations) { - val patterns = patternGenerator.generatePatterns(api, remainingText) - if (patterns.isEmpty()) break - val applyPatterns = applyPatterns(remainingText, (result?.patterns ?: emptyList()) + patterns) - result = applyPatterns.first - remainingText = applyPatterns.second + override fun getSmartParser(api: API): (LogData, String) -> LogData { + val patternGenerator = LogPatternGenerator(parsingModel, temperature) + return { runningDocument, prompt -> + parseText(prompt, patternGenerator, api, runningDocument.patterns ?: emptyList()) + } + } + + private fun parseText( + originalText: String, + patternGenerator: LogPatternGenerator, + api: API, + existingPatterns: List + ): LogData { + var remainingText = originalText + var result: LogData? = null + var iterationCount = 0 + var currentPatterns = existingPatterns + try { + // First try with existing patterns + if (currentPatterns.isNotEmpty()) { + val applyPatterns = applyPatterns(remainingText, currentPatterns) + result = applyPatterns.first + remainingText = applyPatterns.second + } + // Then generate new patterns for remaining text + while (remainingText.isNotBlank() && iterationCount++ < maxIterations) { + val newPatterns = patternGenerator.generatePatterns(api, remainingText) + if (newPatterns.isEmpty()) break + currentPatterns = (currentPatterns + newPatterns).distinctBy { it.regex } + val applyPatterns = applyPatterns(remainingText, currentPatterns) + result = if (result != null) { + merge(result, applyPatterns.first) + } else { + applyPatterns.first } - } catch (e: Exception) { - log.error("Error parsing log data", e) + remainingText = applyPatterns.second } - result ?: LogData() + } catch (e: Exception) { + log.error("Error parsing log data", e) } + return result ?: LogData() } + private fun applyPatterns(text: String, patterns: List): Pair { val patterns = patterns.filter { it.regex != null }.groupBy { it.id }.map { it.value.first() } val matches = patterns.flatMap { pattern -> diff --git a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/parse/TextReader.kt b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/parse/TextReader.kt index 54de29f6..c4afd9ed 100644 --- a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/parse/TextReader.kt +++ b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/parse/TextReader.kt @@ -41,10 +41,10 @@ class TextReader(private val textFile: File) : DocumentParserApp.DocumentReader var fitness = -((leftSize.toDouble() / text.length) * Math.log1p(rightSize.toDouble() / text.length) + (rightSize.toDouble() / text.length) * Math.log1p(leftSize.toDouble() / text.length)) if (lines[i].isEmpty()) fitness *= 2 - i to fitness.toDouble() + i to fitness }.toTypedArray().toMutableList() - var bestSplitIndex = splitFitnesses.minByOrNull { it.second }?.first ?: lines.size / 2 + val bestSplitIndex = splitFitnesses.minByOrNull { it.second }?.first ?: lines.size / 2 val leftText = lines.subList(0, bestSplitIndex).joinToString("\n") val rightText = lines.subList(bestSplitIndex, lines.size).joinToString("\n") return splitIntoPages(leftText, maxChars) + splitIntoPages(rightText, maxChars) diff --git a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/CommandAutoFixTask.kt b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/CommandAutoFixTask.kt index 9038f428..45afc49d 100644 --- a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/CommandAutoFixTask.kt +++ b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/CommandAutoFixTask.kt @@ -60,10 +60,12 @@ ${planSettings.commandAutoFixCommands?.joinToString("\n") { " * ${File(it).na api2: OpenAIClient, planSettings: PlanSettings ) { + var autoRetries = if(planSettings.autoFix) 5 else 0 val semaphore = Semaphore(0) val hasError = AtomicBoolean(false) val onComplete = { semaphore.release() } - Retryable(agent.ui, task = task) { + lateinit var retryable: Retryable + retryable = Retryable(agent.ui, task = task) { val task = agent.ui.newTask(false).apply { it.append(placeholder) } this.planTask?.commands?.forEachIndexed { index, commandWithDir -> val alias = commandWithDir.command.firstOrNull() @@ -122,19 +124,22 @@ ${planSettings.commandAutoFixCommands?.joinToString("\n") { " * ${File(it).na ) } resultFn("All Command Auto Fix tasks completed") - task.add(if (!hasError.get()) { - onComplete() - MarkdownUtil.renderMarkdown("## All Command Auto Fix tasks completed successfully\n", ui = agent.ui, tabs = false) - } else { - MarkdownUtil.renderMarkdown( - "## Some Command Auto Fix tasks failed\n", - ui = agent.ui - ) + acceptButtonFooter( - agent.ui - ) { + task.add( + if (!hasError.get()) { onComplete() - } - }) + MarkdownUtil.renderMarkdown("## All Command Auto Fix tasks completed successfully\n", ui = agent.ui, tabs = false) + } else { + val s = MarkdownUtil.renderMarkdown( + "## Some Command Auto Fix tasks failed\n", + ui = agent.ui + ) + acceptButtonFooter( + agent.ui + ) { + onComplete() + } + if(autoRetries-- > 0) retryable.retry() + s + }) task.placeholder } try { diff --git a/webui/src/main/resources/application/README.md b/webui/src/main/resources/application/README.md new file mode 100644 index 00000000..201c0095 --- /dev/null +++ b/webui/src/main/resources/application/README.md @@ -0,0 +1,201 @@ +# Web UI JavaScript Architecture + +## Overview + +This is a modular JavaScript application that provides a web-based chat interface with real-time WebSocket communication, theme customization, and dynamic UI +components. The architecture emphasizes performance, maintainability, and user experience. + +### Key Features + +* Real-time bidirectional communication via WebSockets +* Multiple theme support with persistent preferences +* Dynamic content loading and caching +* Responsive UI with accessibility features +* Robust error handling and recovery + +## Quick Start + +```javascript +// Initialize the application +import {connect} from './chat.js'; +import {setupUIHandlers} from './uiHandlers.js'; +import {getSessionId} from './functions.js'; + +// Set up WebSocket connection +const sessionId = getSessionId(); +connect(sessionId, handleWebSocketMessage); + +// Configure UI +setupUIHandlers(); +``` + +## Core Modules + +### main.js + +The application entry point that coordinates initialization and setup: + +* Manages DOM content loading +* Initializes WebSocket connection +* Sets up UI event handlers +* Configures theme and appearance + +Example usage: + +```javascript +document.addEventListener('DOMContentLoaded', () => { + setupUIHandlers(); + setupMessageInput(form, messageInput); + connect(sessionId, handleWebSocketMessage); + fetchAppConfig(sessionId).then(config => { + applyConfiguration(config); + }); +}); +``` + +### chat.js + +Manages WebSocket communication with features: + +* Automatic reconnection with exponential backoff +* Message queueing during disconnections +* Custom message handlers +* Connection state management + +Example usage: + +```javascript +// Send a message +queueMessage('Hello world'); + +// Connect with custom handler +connect(sessionId, (event) => { + console.log('Received:', event.data); +}); +// Handle disconnection +socket.addEventListener('close', () => { + showDisconnectedOverlay(true); + reconnect(sessionId, customReceiveFunction); +}); +``` + +### functions.js + +Core utility functions for common operations: + +* DOM element caching +* Modal management +* Session handling +* UI state management + +Example usage: + +```javascript +// Show a modal +showModal('settings'); + +// Get cached DOM element +const element = getCachedElement('modal-content'); +// Toggle verbose mode +toggleVerbose(); +``` + +### messageHandling.js + +Message processing functionality with support for: + +* Versioned messages +* Message substitution +* Depth-limited recursion +* Timeout protection + +Message format: + +```javascript +// Format: messageId,version,content +// Example: m1,1,Hello world +// Special prefixes: +// u* * User messages (e.g., u1, u2) +// z* * System messages (e.g., z1, z2) +// m* * Application messages (e.g., m1, m2) +``` + +### uiHandlers.js + +UI event handling system with support for: + +* Event delegation +* Action dispatching +* Keyboard shortcuts +* Modal management + +Supported actions: + +* userTxt: Submit user text input + +## Supporting Modules + +### appConfig.js + +Application configuration management with: + +* Runtime configuration updates +* Persistent settings +* Dynamic UI adaptation + +Configuration options: + +```javascript +{ + singleInput : boolean, // Single/multiple input mode + stickyInput : boolean, // Input stays visible + loadImages : boolean, // Auto-load images + showMenubar : boolean, // Show/hide menubar + applicationName : string // Custom application title +} +``` + +### tabs.js + +Tab system implementation featuring: + +* Persistent tab state +* Nested tabs support +* Dynamic tab updates +* Memory efficient caching + +HTML structure: + +```html + +
+ +
Content 1
+ +
+ +
Nested Content 1
+
+
+``` + +### theme.js + +Theme management: +Available themes: + +* main: Default theme +* night: Dark mode +* forest: Nature-inspired +* pony: Playful theme +* alien: Sci-fi theme + +### uiSetup.js + +UI initialization: +Features: + +* Auto-expanding textareas +* Enter to submit (Shift+Enter for newline) +* Loading state indicators +* User authentication state management diff --git a/webui/src/main/resources/application/main.js b/webui/src/main/resources/application/main.js index cc16dee3..7526f6ad 100644 --- a/webui/src/main/resources/application/main.js +++ b/webui/src/main/resources/application/main.js @@ -89,26 +89,9 @@ document.addEventListener('DOMContentLoaded', () => { setupUserInfo(loginLink, usernameLink, userSettingsLink, userUsageLink, logoutLink); fetchAppConfig(sessionId) - .then(({singleInput, stickyInput, loadImages, showMenubar}) => { - // Use the config values as needed - console.log('App config loaded:', {singleInput, stickyInput, loadImages, showMenubar}); - }) .catch(error => { console.error('There was a problem with the fetch operation:', error); }); - updateTabs(); - Array.from(document.getElementsByClassName('tabs-container')).forEach(tabsContainer => { - console.log('Restoring tabs for container:', tabsContainer.id); - const savedTab = localStorage.getItem(`selectedTab_${tabsContainer.id}`); - if (savedTab) { - const savedButton = tabsContainer.querySelector(`[data-for-tab="${savedTab}"]`); - console.log('Main script finished loading'); - if (savedButton) { - savedButton.click(); - console.log(`Restored saved tab: ${savedTab}`); - } - } - }); }); \ No newline at end of file diff --git a/webui/src/main/resources/application/tabs.js b/webui/src/main/resources/application/tabs.js index ebdf80f9..1f210cc3 100644 --- a/webui/src/main/resources/application/tabs.js +++ b/webui/src/main/resources/application/tabs.js @@ -4,7 +4,7 @@ const tabCache = new Map(); let isRestoringTabs = false; const MAX_RECURSION_DEPTH = 10; const OPERATION_TIMEOUT = 5000; // 5 seconds -function setActiveTab(button, tabsContainer) { +function setActiveTab(button, tabsContainer, depth = 0) { const forTab = button.getAttribute('data-for-tab'); const tabsContainerId = tabsContainer.id; if (button.classList.contains('active')) return; @@ -20,7 +20,7 @@ function setActiveTab(button, tabsContainer) { if (content.getAttribute('data-tab') === forTab) { content.classList.add('active'); content.style.display = 'block'; - updateNestedTabs(content, 0); + updateNestedTabs(content, depth + 1); } else { content.classList.remove('active'); content.style.display = 'none'; @@ -31,19 +31,22 @@ function setActiveTab(button, tabsContainer) { export function updateTabs() { const tabButtons = document.querySelectorAll('.tab-button'); const tabsContainers = new Set(); - const clickHandler = (event) => { - event.stopPropagation(); - const button = event.currentTarget; - const tabsContainer = button.closest('.tabs-container'); - setActiveTab(button, tabsContainer); - }; tabButtons.forEach(button => { const tabsContainer = button.closest('.tabs-container'); tabsContainers.add(tabsContainer); - if (button.hasListener) return; - button.hasListener = true; - button.addEventListener('click', clickHandler); + }); + + tabsContainers.forEach(tabsContainer => { + if (tabsContainer.hasListener) return; + tabsContainer.hasListener = true; + tabsContainer.addEventListener('click', (event) => { + const button = event.target.closest('.tab-button'); + if (button && tabsContainer.contains(button)) { + setActiveTab(button, tabsContainer, 0); + event.stopPropagation(); + } + }); }); // Restore the selected tabs from localStorage @@ -53,9 +56,11 @@ export function updateTabs() { requestAnimationFrame(() => { const savedTab = getSavedTab(tabsContainer.id); const buttonToActivate = savedTab - ? tabsContainer.querySelector(`.tab-button[data-for-tab="${savedTab}"]`) + ? tabsContainer.querySelector(`.tab-button[data-for-tab="${CSS.escape(savedTab)}"]`) : tabsContainer.querySelector('.tab-button'); - buttonToActivate?.click(); + if (buttonToActivate) { + buttonToActivate.click(); + } }); }); isRestoringTabs = false; @@ -74,8 +79,8 @@ function getSavedTab(containerId) { } } -function updateNestedTabs(element, depth) { - if (depth > MAX_RECURSION_DEPTH) { +function updateNestedTabs(element, depth = 0) { + if (depth >= MAX_RECURSION_DEPTH) { console.warn('Max recursion depth reached in updateNestedTabs'); return; } @@ -101,15 +106,19 @@ function updateNestedTabs(element, depth) { ? tabsContainer.querySelector(`.tab-button[data-for-tab="${activeTab}"]`) : tabsContainer.querySelector('.tab-button'); if (buttonToActivate) requestAnimationFrame(() => buttonToActivate.click()); + const savedTab = getSavedTab(tabsContainer.id); + const savedButton = savedTab + ? tabsContainer.querySelector(`.tab-button[data-for-tab="${CSS.escape(savedTab)}"]`) + : null; + if (savedButton && !savedButton.classList.contains('active')) { + requestAnimationFrame(() => savedButton.click()); + } } - const savedTab = getSavedTab(tabsContainer.id); - const savedButton = savedTab && tabsContainer.querySelector(`.tab-button[data-for-tab="${savedTab}"]`); - if (savedButton && !savedButton.classList.contains('active')) requestAnimationFrame(() => savedButton.click()); } catch (e) { console.warn('Failed to update nested tabs:', e); } + clearTimeout(timeoutId); } - clearTimeout(timeoutId); } document.addEventListener('DOMContentLoaded', () => {