1.0.73 (#78)

* 1.0.73 * tabs * tabs and headless stubs * wip * Update DataStorage.kt
SimiaCryptus · May 24, 2024 · de7ffb8 · de7ffb8
1 parent e062167
commit de7ffb8
Show file tree

Hide file tree

Showing 25 changed files with 212 additions and 106 deletions.
diff --git a/README.md b/README.md
@@ -76,18 +76,18 @@ Maven:
 <dependency>
     <groupId>com.simiacryptus</groupId>
     <artifactId>skyenet-webui</artifactId>
-    <version>1.0.56</version>
+    <version>1.0.57</version>
 </dependency>
 ```
 
 Gradle:
 
 ```groovy
-implementation group: 'com.simiacryptus', name: 'skyenet', version: '1.0.56'
+implementation group: 'com.simiacryptus', name: 'skyenet', version: '1.0.57'
 ```
 
 ```kotlin
-implementation("com.simiacryptus:skyenet:1.0.56")
+implementation("com.simiacryptus:skyenet:1.0.57")
 ```
 
 ### 🌟 To Use

diff --git a/core/build.gradle.kts b/core/build.gradle.kts
@@ -32,7 +32,7 @@ val jackson_version = "2.17.0"
 
 dependencies {
 
-    implementation(group = "com.simiacryptus", name = "jo-penai", version = "1.0.56")
+    implementation(group = "com.simiacryptus", name = "jo-penai", version = "1.0.57")
 
     implementation("org.apache.commons:commons-text:1.11.0")
 

diff --git a/core/src/main/kotlin/com/simiacryptus/skyenet/core/actors/ActorSystem.kt b/core/src/main/kotlin/com/simiacryptus/skyenet/core/actors/ActorSystem.kt
@@ -66,10 +66,8 @@ open class ActorSystem<T : Enum<*>>(
     private fun getWrapper(name: String) = synchronized(wrapperMap) {
         wrapperMap.getOrPut(name) {
             FunctionWrapper(JsonFunctionRecorder(
-                File(
-                    ApplicationServices.dataStorageRoot,
-                    "${if (session.isGlobal()) "global" else user}/$session/actors/$name"
-                ).apply { mkdirs() }))
+                dataStorage.getSessionDir(user, session).resolve("actors/$name").apply { mkdirs() }
+            ))
         }
     }
 

diff --git a/core/src/main/kotlin/com/simiacryptus/skyenet/core/platform/ApplicationServices.kt b/core/src/main/kotlin/com/simiacryptus/skyenet/core/platform/ApplicationServices.kt
@@ -7,7 +7,6 @@ import com.simiacryptus.jopenai.ApiModel
 import com.simiacryptus.jopenai.models.APIProvider
 import com.simiacryptus.jopenai.models.ChatModels
 import com.simiacryptus.jopenai.models.OpenAIModel
-import com.simiacryptus.jopenai.util.JsonUtil
 import com.simiacryptus.skyenet.core.platform.file.*
 import com.simiacryptus.skyenet.core.util.Selenium
 import java.io.File
@@ -127,7 +126,8 @@ interface StorageInterface {
     ): Date?
 
     fun listSessions(
-        user: User?
+        user: User?,
+        path: String,
     ): List<Session>
 
     fun <T : Any> setJson(
@@ -144,7 +144,7 @@ interface StorageInterface {
         value: String
     )
 
-    fun listSessions(dir: File): List<String>
+    fun listSessions(dir: File, path: String): List<String>
     fun userRoot(user: User?): File
     fun deleteSession(user: User?, session: Session)
     fun getMessageIds(

diff --git a/core/src/main/kotlin/com/simiacryptus/skyenet/core/platform/ClientManager.kt b/core/src/main/kotlin/com/simiacryptus/skyenet/core/platform/ClientManager.kt
@@ -118,7 +118,7 @@ open class ClientManager {
             null, user, OperationType.GlobalKey
         )
         if (!canUseGlobalKey) throw RuntimeException("No API key")
-        val logfile = dataStorageRoot?.resolve("${if (session.isGlobal()) "global" else user}/$session/openai.log")
+        val logfile = dataStorageRoot?.resolve("${if (session.isGlobal()) "global" else "user-sessions/$user"}/$session/openai.log")
             ?.apply { parentFile?.mkdirs() }
         logfile?.parentFile?.mkdirs()
         return (if (ClientUtil.keyMap.isNotEmpty()) {

diff --git a/core/src/main/kotlin/com/simiacryptus/skyenet/core/platform/file/DataStorage.kt b/core/src/main/kotlin/com/simiacryptus/skyenet/core/platform/file/DataStorage.kt
@@ -1,14 +1,15 @@
 package com.simiacryptus.skyenet.core.platform.file
 
 import com.simiacryptus.jopenai.util.JsonUtil
-import com.simiacryptus.skyenet.core.platform.ApplicationServices
 import com.simiacryptus.skyenet.core.platform.Session
 import com.simiacryptus.skyenet.core.platform.StorageInterface
 import com.simiacryptus.skyenet.core.platform.StorageInterface.Companion.validateSessionId
 import com.simiacryptus.skyenet.core.platform.User
 import java.io.File
 import java.text.SimpleDateFormat
 import java.util.*
+import kotlin.reflect.jvm.javaType
+import kotlin.reflect.typeOf
 
 open class DataStorage(
     private val dataDir: File
@@ -21,7 +22,7 @@ open class DataStorage(
         validateSessionId(session)
         log.debug("Fetching messages for session: ${session.sessionId}, user: ${user?.email}")
         val messageDir =
-            ApplicationServices.dataStorageRoot.resolve("${if (session.isGlobal()) "global" else user}/$session/messages/")
+            getSessionDir(user, session).resolve("messages/")
                 .apply { mkdirs() }
         val messages = LinkedHashMap<String, String>()
         getMessageIds(user, session).forEach { messageId ->
@@ -48,8 +49,8 @@ open class DataStorage(
         return when (parts.size) {
             3 -> {
                 val root = when {
-                    parts[0] == "G" -> dataDir
-                    parts[0] == "U" -> userRoot(user)
+                    parts[0] == "G" -> dataDir.resolve("global")
+                    parts[0] == "U" -> dataDir.resolve("user-sessions/$user")
                     else -> throw IllegalArgumentException("Invalid session ID: $session")
                 }
                 val dateDir = File(root, parts[1])
@@ -59,8 +60,8 @@ open class DataStorage(
             }
 
             2 -> {
-                val dateDir = File(dataDir, parts[0])
-                val sessionDir = File(dateDir, parts[1])
+                val dateDir = dataDir.resolve("global").resolve(parts[0])
+                val sessionDir = dateDir.resolve(parts[1])
                 log.debug("Session directory for session: ${session.sessionId} is ${sessionDir.absolutePath}")
                 sessionDir
             }
@@ -105,10 +106,7 @@ open class DataStorage(
         log.debug("Fetching message IDs for session: ${session.sessionId}, user: ${user?.email}")
         val sessionDir = getSessionDir(user, session)
         val settings = run {
-            val settingsFile = File(
-                ApplicationServices.dataStorageRoot,
-                "${if (session.isGlobal()) "global" else user}/$session/internal.json"
-            )
+            val settingsFile = sessionDir.resolve("internal.json")
             if (!settingsFile.exists()) null else {
                 JsonUtil.objectMapper().readValue(settingsFile, Map::class.java) as Map<*, *>
             }
@@ -117,8 +115,8 @@ open class DataStorage(
         val ids = messageFiles(session, user).entries.sortedBy { it.key.lastModified() }
             .map { it.key.nameWithoutExtension }.toList()
         setJson(
-            ApplicationServices.dataStorageRoot,
-            "${if (session.isGlobal()) "global" else user}/$session/internal.json",
+            sessionDir,
+            "internal.json",
             settings.plus("ids" to ids.joinToString(","))
         )
         log.debug("Message IDs for session: ${session.sessionId} are $ids")
@@ -132,18 +130,16 @@ open class DataStorage(
     ) {
         validateSessionId(session)
         log.debug("Setting message IDs for session: ${session.sessionId}, user: ${user?.email} to $ids")
+        val sessionDir = getSessionDir(user, session)
         val settings = run {
-            val settingsFile = File(
-                ApplicationServices.dataStorageRoot,
-                "${if (session.isGlobal()) "global" else user}/$session/internal.json"
-            )
+            val settingsFile = sessionDir.resolve("internal.json")
             if (!settingsFile.exists()) null else {
                 JsonUtil.objectMapper().readValue(settingsFile, Map::class.java) as Map<*, *>
             }
         } ?: mapOf<String, String>()
         setJson(
-            ApplicationServices.dataStorageRoot,
-            "${if (session.isGlobal()) "global" else user}/$session/internal.json",
+            sessionDir,
+            "internal.json",
             settings.plus("ids" to ids.joinToString(","))
         )
     }
@@ -154,8 +150,8 @@ open class DataStorage(
     ): Date? {
         validateSessionId(session)
         log.debug("Fetching session time for session: ${session.sessionId}, user: ${user?.email}")
-        val settingsFile =
-            ApplicationServices.dataStorageRoot.resolve("${if (session.isGlobal()) "global" else user}/$session/internal.json")
+        val sessionDir = getSessionDir(user, session)
+        val settingsFile = sessionDir.resolve("internal.json")
         val settings = run {
             if (!settingsFile.exists()) null else {
                 JsonUtil.objectMapper().readValue(settingsFile, Map::class.java) as Map<*, *>
@@ -168,8 +164,8 @@ open class DataStorage(
         return if (null != file) {
             val date = Date(file.lastModified())
             setJson(
-                ApplicationServices.dataStorageRoot,
-                "${if (session.isGlobal()) "global" else user}/$session/internal.json",
+                sessionDir,
+                "internal.json",
                 settings.plus("time" to dateFormat.format(date))
             )
             log.debug("Session time for session: ${session.sessionId} is $date")
@@ -183,8 +179,9 @@ open class DataStorage(
     private fun messageFiles(
         session: Session,
         user: User?,
-    ) =
-        ApplicationServices.dataStorageRoot.resolve("${if (session.isGlobal()) "global" else user}/$session/messages")
+    ): Map<File, String> {
+
+        return getSessionDir(user, session).resolve("messages")
             .apply { mkdirs() }.listFiles()
             ?.filter { file -> file.isFile }
             ?.map { messageFile ->
@@ -203,13 +200,23 @@ open class DataStorage(
                     }
                 }
             }?.filter { it.second.isNotEmpty() }?.toList()?.toMap() ?: mapOf()
+    }
 
     override fun listSessions(
-        user: User?
+        user: User?,
+        path: String
     ): List<Session> {
         log.debug("Listing sessions for user: ${user?.email}")
-        val globalSessions = listSessions(dataDir)
-        val userSessions = if (user == null) listOf() else listSessions(userRoot(user))
+        val globalSessions = listSessions(dataDir.resolve("global"), path)
+        val userSessions = if (user == null) listOf() else listSessions(
+            dataDir.resolve("user-sessions").resolve(
+                if (user?.email != null) {
+                    user.email
+                } else {
+                    throw IllegalArgumentException("User required for private session")
+                }
+            ).apply { mkdirs() }, path
+        )
         log.debug("Found ${globalSessions.size} global sessions and ${userSessions.size} user sessions for user: ${user?.email}")
         return ((globalSessions.map {
             try {
@@ -235,8 +242,7 @@ open class DataStorage(
 
     private fun <T : Any> setJson(sessionDir: File, filename: String, settings: T): T {
         log.debug("Setting JSON for session directory: ${sessionDir.absolutePath}, filename: $filename")
-        val settingsFile = File(sessionDir, filename)
-        settingsFile.parentFile.mkdirs()
+        val settingsFile = sessionDir.resolve(filename).apply { parentFile.mkdirs() }
         JsonUtil.objectMapper().writeValue(settingsFile, settings)
         return settings
     }
@@ -250,7 +256,7 @@ open class DataStorage(
         validateSessionId(session)
         log.debug("Updating message for session: ${session.sessionId}, messageId: $messageId, user: ${user?.email}")
         val file =
-            ApplicationServices.dataStorageRoot.resolve("${if (session.isGlobal()) "global" else user}/$session/messages/$messageId.json")
+            getSessionDir(user, session).resolve("messages/$messageId.json")
                 .apply { parentFile.mkdirs() }
         if (!file.exists()) {
             file.parentFile.mkdirs()
@@ -270,32 +276,28 @@ open class DataStorage(
         }
     }
 
-    override fun listSessions(dir: File): List<String> {
+    override fun listSessions(dir: File, path: String): List<String> {
         log.debug("Listing sessions in directory: ${dir.absolutePath}")
-        val files = dir.listFiles()?.flatMap { it.listFiles()?.toList() ?: listOf() }?.filter { sessionDir ->
-            val operationDir = File(sessionDir, "messages")
-            if (!operationDir.exists()) false else {
-                val listFiles = operationDir.listFiles()?.flatMap {
-                    when {
-                        it.isDirectory -> it.listFiles()?.toList() ?: listOf()
-                        else -> listOf(it)
-                    }
-                }?.filter { it.isFile && !it.name.startsWith("aaa") }
-                (listFiles?.size ?: 0) > 0
-            }
-        }?.sortedBy { it.lastModified() } ?: listOf()
+        val files = dir.listFiles()
+            ?.flatMap { it.listFiles()?.toList() ?: listOf() }
+            ?.filter { sessionDir ->
+                val resolve = sessionDir.resolve("info.json")
+                if (!resolve.exists()) return@filter false
+                val infoJson = resolve.readText()
+                val infoData = JsonUtil.fromJson<Map<String, String>>(infoJson, typeOf<Map<String, String>>().javaType)
+                path == infoData["path"]
+            }?.sortedBy { it.lastModified() } ?: listOf()
         log.debug("Found ${files.size} sessions in directory: ${dir.absolutePath}")
         return files.map { it.parentFile.name + "-" + it.name }
     }
 
-    override fun userRoot(user: User?) = File(
-        File(dataDir, "users"),
+    override fun userRoot(user: User?) = dataDir.resolve("users").resolve(
         if (user?.email != null) {
             user.email
         } else {
             throw IllegalArgumentException("User required for private session")
         }
-    )
+    ).apply { mkdirs() }
 
     override fun deleteSession(user: User?, session: Session) {
         validateSessionId(session)

diff --git a/core/src/main/kotlin/com/simiacryptus/skyenet/core/platform/test/StorageInterfaceTest.kt b/core/src/main/kotlin/com/simiacryptus/skyenet/core/platform/test/StorageInterfaceTest.kt
@@ -92,7 +92,7 @@ abstract class StorageInterfaceTest(val storage: StorageInterface) {
         val user = User(email = "[email protected]")
 
         // Act
-        val sessions = storage.listSessions(user)
+        val sessions = storage.listSessions(user, "",)
 
         // Assert
         Assertions.assertNotNull(sessions)
@@ -138,7 +138,7 @@ abstract class StorageInterfaceTest(val storage: StorageInterface) {
         val directory = File(System.getProperty("user.dir")) // Example directory
 
         // Act
-        val sessionList = storage.listSessions(directory)
+        val sessionList = storage.listSessions(directory, "")
 
         // Assert
         Assertions.assertNotNull(sessionList)

diff --git a/gradle.properties b/gradle.properties
@@ -1,4 +1,4 @@
 # Gradle Releases -> https://github.com/gradle/gradle/releases
 libraryGroup = com.simiacryptus.skyenet
-libraryVersion = 1.0.72
+libraryVersion = 1.0.73
 gradleVersion = 7.6.1
diff --git a/webui/build.gradle.kts b/webui/build.gradle.kts
@@ -35,7 +35,7 @@ val jetty_version = "11.0.18"
 val jackson_version = "2.17.0"
 dependencies {
 
-    implementation(group = "com.simiacryptus", name = "jo-penai", version = "1.0.56")
+    implementation(group = "com.simiacryptus", name = "jo-penai", version = "1.0.57")
 
     implementation(project(":core"))
     implementation(project(":kotlin"))

diff --git a/webui/src/main/kotlin/com/simiacryptus/skyenet/AgentPatterns.kt b/webui/src/main/kotlin/com/simiacryptus/skyenet/AgentPatterns.kt
@@ -1,6 +1,7 @@
 package com.simiacryptus.skyenet
 
 import com.simiacryptus.skyenet.webui.application.ApplicationInterface
+import java.util.UUID
 
 object AgentPatterns {
 
@@ -12,15 +13,15 @@ object AgentPatterns {
         val tasks = map.entries.map { (key, value) ->
             key to ui.newTask(root = false)
         }.toMap()
-        ui.socketManager.scheduledThreadPoolExecutor.schedule({
+        ui.socketManager?.scheduledThreadPoolExecutor?.schedule({
             tasks.forEach { (key, task) ->
                 task.complete(map[key]!!)
             }
         }, 200, java.util.concurrent.TimeUnit.MILLISECONDS)
         displayMapInTabs(tasks.mapValues { it.value.placeholder }, ui = ui, split = false)
     } else {
         """
-    |<div class="tabs-container">
+    |<div class="tabs-container" id="${UUID.randomUUID()}">
     |<div class="tabs">
     |${
             map.keys.joinToString("\n") { key ->

diff --git a/webui/src/main/kotlin/com/simiacryptus/skyenet/TabbedDisplay.kt b/webui/src/main/kotlin/com/simiacryptus/skyenet/TabbedDisplay.kt
@@ -1,6 +1,7 @@
 package com.simiacryptus.skyenet
 
 import com.simiacryptus.skyenet.webui.session.SessionTask
+import java.util.*
 
 open class TabbedDisplay(
     val task: SessionTask,
@@ -14,7 +15,7 @@ open class TabbedDisplay(
 
     val size: Int get() = tabs.size
     open fun render() = """
-    <div class="tabs-container">
+    <div class="tabs-container" id="${UUID.randomUUID()}">
       ${renderTabButtons()}
       ${
         tabs.withIndex().joinToString("\n")

diff --git a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/coding/CodingAgent.kt b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/coding/CodingAgent.kt
@@ -83,8 +83,8 @@ open class CodingAgent<T : Interpreter>(
         task.complete(newTask.placeholder)
         Retryable(ui, newTask) {
             val newTask = ui.newTask(root = false)
-            ui.socketManager.scheduledThreadPoolExecutor.schedule({
-                ui.socketManager.pool.submit {
+            ui.socketManager?.scheduledThreadPoolExecutor!!.schedule({
+                ui.socketManager?.pool?.submit {
                     val statusSB = newTask.add("Running...")
                     displayCode(newTask, codeRequest)
                     statusSB?.clear()

diff --git a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/general/WebDevApp.kt b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/general/WebDevApp.kt
@@ -365,7 +365,7 @@ class WebDevAgent(
             },
             outputFn = { code ->
                 renderMarkdown(
-                    ui.socketManager.addApplyFileDiffLinks(
+                    ui.socketManager!!.addApplyFileDiffLinks(
                         root = root.toPath(),
                         code = {
                             codeFiles.filter {