diff --git a/core/src/main/kotlin/com/simiacryptus/skyenet/core/actors/ActorSystem.kt b/core/src/main/kotlin/com/simiacryptus/skyenet/core/actors/ActorSystem.kt index 8f86d07a..b32fe92c 100644 --- a/core/src/main/kotlin/com/simiacryptus/skyenet/core/actors/ActorSystem.kt +++ b/core/src/main/kotlin/com/simiacryptus/skyenet/core/actors/ActorSystem.kt @@ -5,7 +5,6 @@ import com.simiacryptus.skyenet.core.platform.ApplicationServices import com.simiacryptus.skyenet.core.platform.Session import com.simiacryptus.skyenet.core.platform.StorageInterface import com.simiacryptus.skyenet.core.platform.User -import com.simiacryptus.skyenet.core.platform.file.DataStorage.Companion.SYS_DIR import com.simiacryptus.skyenet.core.util.FunctionWrapper import com.simiacryptus.skyenet.core.util.JsonFunctionRecorder import java.io.File @@ -68,7 +67,7 @@ open class ActorSystem>( wrapperMap.getOrPut(name) { FunctionWrapper(JsonFunctionRecorder( File( - SYS_DIR, + ApplicationServices.dataStorageRoot, "${if (session.isGlobal()) "global" else user}/$session/actors/$name" ).apply { mkdirs() })) } diff --git a/core/src/main/kotlin/com/simiacryptus/skyenet/core/actors/CodingActor.kt b/core/src/main/kotlin/com/simiacryptus/skyenet/core/actors/CodingActor.kt index bc02648c..f1d383ce 100644 --- a/core/src/main/kotlin/com/simiacryptus/skyenet/core/actors/CodingActor.kt +++ b/core/src/main/kotlin/com/simiacryptus/skyenet/core/actors/CodingActor.kt @@ -73,12 +73,15 @@ open class CodingActor( |Each response should have EXACTLY ONE code block. Do not use inline blocks. |$formatInstructions | - |Defined symbols include {${symbols.keys.joinToString(", ")}} described below: + |Defined symbols include ${symbols.keys.joinToString(", ")} described below: | |```${this.describer.markupLanguage} |${this.apiDescription} |``` | + |DO NOT RE-DEFINE THESE SYMBOLS: ${symbols.keys.joinToString(", ")} + |They are already defined for you. + | |${details ?: ""} |""".trimMargin().trim() } else """ @@ -475,12 +478,12 @@ open class CodingActor( fun errorMessage(ex: ScriptException, code: String) = try { """ - |```text - |${ex.message ?: ""} at line ${ex.lineNumber} column ${ex.columnNumber} - | ${if (ex.lineNumber > 0) code.split("\n")[ex.lineNumber - 1] else ""} - | ${if (ex.columnNumber > 0) " ".repeat(ex.columnNumber - 1) + "^" else ""} - |``` - """.trimMargin().trim() + |```text + |${ex.message ?: ""} at line ${ex.lineNumber} column ${ex.columnNumber} + | ${if (ex.lineNumber > 0) code.split("\n")[ex.lineNumber - 1] else ""} + | ${if (ex.columnNumber > 0) " ".repeat(ex.columnNumber - 1) + "^" else ""} + |``` + """.trimMargin().trim() } catch (_: Exception) { ex.message ?: "" } diff --git a/core/src/main/kotlin/com/simiacryptus/skyenet/core/platform/ApplicationServices.kt b/core/src/main/kotlin/com/simiacryptus/skyenet/core/platform/ApplicationServices.kt index 607780f4..b84b49d3 100644 --- a/core/src/main/kotlin/com/simiacryptus/skyenet/core/platform/ApplicationServices.kt +++ b/core/src/main/kotlin/com/simiacryptus/skyenet/core/platform/ApplicationServices.kt @@ -66,7 +66,7 @@ object ApplicationServices { require(!isLocked) { "ApplicationServices is locked" } field = value } - var usageManager: UsageInterface = UsageManager(File(dataStorageRoot, ".skyenet/usage")) + var usageManager: UsageInterface = UsageManager(File(dataStorageRoot, "usage")) set(value) { require(!isLocked) { "ApplicationServices is locked" } field = value diff --git a/core/src/main/kotlin/com/simiacryptus/skyenet/core/platform/ClientManager.kt b/core/src/main/kotlin/com/simiacryptus/skyenet/core/platform/ClientManager.kt index 6d34ef54..736479ba 100644 --- a/core/src/main/kotlin/com/simiacryptus/skyenet/core/platform/ClientManager.kt +++ b/core/src/main/kotlin/com/simiacryptus/skyenet/core/platform/ClientManager.kt @@ -9,10 +9,13 @@ import com.simiacryptus.jopenai.OpenAIClient import com.simiacryptus.jopenai.models.APIProvider import com.simiacryptus.jopenai.models.OpenAIModel import com.simiacryptus.jopenai.util.ClientUtil +import com.simiacryptus.skyenet.core.platform.ApplicationServices.dataStorageFactory +import com.simiacryptus.skyenet.core.platform.ApplicationServices.dataStorageRoot +import com.simiacryptus.skyenet.core.platform.ApplicationServices.userSettingsManager import com.simiacryptus.skyenet.core.platform.AuthorizationInterface.OperationType -import com.simiacryptus.skyenet.core.platform.file.DataStorage.Companion.SYS_DIR import org.apache.hc.client5.http.impl.classic.CloseableHttpClient import org.apache.hc.core5.http.HttpRequest +import org.slf4j.LoggerFactory import org.slf4j.event.Level import java.io.File import java.util.concurrent.* @@ -30,6 +33,7 @@ open class ClientManager { user: User?, dataStorage: StorageInterface?, ): OpenAIClient { + log.debug("Fetching client for session: {}, user: {}", session, user) val key = SessionKey(session, user) return if (null == dataStorage) clientCache[key] ?: throw IllegalStateException("No data storage") else clientCache.getOrPut(key) { createClient(session, user, dataStorage)!! } @@ -42,15 +46,17 @@ open class ClientManager { SynchronousQueue(), RecordingThreadFactory(session, user) ) + /*createScheduledPool*/ protected open fun createScheduledPool(session: Session, user: User?, dataStorage: StorageInterface?) = - MoreExecutors.listeningDecorator(ScheduledThreadPoolExecutor(1,)) + MoreExecutors.listeningDecorator(ScheduledThreadPoolExecutor(1)) fun getPool( session: Session, user: User?, dataStorage: StorageInterface?, ): ThreadPoolExecutor { + log.debug("Fetching thread pool for session: {}, user: {}", session, user) val key = SessionKey(session, user) return poolCache.getOrPut(key) { createPool(session, user, dataStorage) @@ -62,6 +68,7 @@ open class ClientManager { user: User?, dataStorage: StorageInterface?, ): ListeningScheduledExecutorService { + log.debug("Fetching scheduled pool for session: {}, user: {}", session, user) val key = SessionKey(session, user) return scheduledPoolCache.getOrPut(key) { createScheduledPool(session, user, dataStorage) @@ -69,12 +76,13 @@ open class ClientManager { } inner class RecordingThreadFactory( - session: Session, - user: User? + val session: Session, + val user: User? ) : ThreadFactory { private val inner = ThreadFactoryBuilder().setNameFormat("Session $session; User $user; #%d").build() val threads = mutableSetOf() override fun newThread(r: Runnable): Thread { + log.debug("Creating new thread for session: {}, user: {}", session, user) inner.newThread(r).also { threads.add(it) return it @@ -87,10 +95,13 @@ open class ClientManager { user: User?, dataStorage: StorageInterface?, ): OpenAIClient? { + log.debug("Creating client for session: {}, user: {}", session, user) if (user != null) { - val userSettings = ApplicationServices.userSettingsManager.getUserSettings(user) - val logfile = SYS_DIR?.resolve("${if (session.isGlobal()) "global" else user}/$session/openai.log")?.apply { parentFile?.mkdirs() } + val userSettings = userSettingsManager.getUserSettings(user) + + val logfile = dataStorageFactory(dataStorageRoot).getSessionDir(user, session).resolve("openai.log").apply { mkdirs() }.resolve("openai.log") logfile?.parentFile?.mkdirs() + log.debug("Logfile: {}", logfile) val userApi = if (userSettings.apiKeys.isNotEmpty()) MonitoredClient( @@ -107,7 +118,8 @@ open class ClientManager { null, user, OperationType.GlobalKey ) if (!canUseGlobalKey) throw RuntimeException("No API key") - val logfile = SYS_DIR?.resolve("${if (session.isGlobal()) "global" else user}/$session/openai.log")?.apply { parentFile?.mkdirs() } + val logfile = dataStorageRoot?.resolve("${if (session.isGlobal()) "global" else user}/$session/openai.log") + ?.apply { parentFile?.mkdirs() } logfile?.parentFile?.mkdirs() return (if (ClientUtil.keyMap.isNotEmpty()) { MonitoredClient( @@ -144,16 +156,26 @@ open class ClientManager { ) { var budget = 2.00 override fun authorize(request: HttpRequest, apiProvider: APIProvider) { + log.debug("Authorizing request for session: {}, user: {}, apiProvider: {}", session, user, apiProvider) require(budget > 0.0) { "Budget Exceeded" } super.authorize(request, ClientUtil.defaultApiProvider) } override fun onUsage(model: OpenAIModel?, tokens: ApiModel.Usage) { + log.debug( + "Usage recorded for session: {}, user: {}, model: {}, tokens: {}", + session, + user, + model, + tokens + ) ApplicationServices.usageManager.incrementUsage(session, user, model!!, tokens) budget -= tokens.cost ?: 0.0 super.onUsage(model, tokens) } } - companion object + companion object { + private val log = LoggerFactory.getLogger(ClientManager::class.java) + } } \ No newline at end of file diff --git a/core/src/main/kotlin/com/simiacryptus/skyenet/core/platform/file/DataStorage.kt b/core/src/main/kotlin/com/simiacryptus/skyenet/core/platform/file/DataStorage.kt index 6e77c41a..c39cc164 100644 --- a/core/src/main/kotlin/com/simiacryptus/skyenet/core/platform/file/DataStorage.kt +++ b/core/src/main/kotlin/com/simiacryptus/skyenet/core/platform/file/DataStorage.kt @@ -1,6 +1,7 @@ package com.simiacryptus.skyenet.core.platform.file import com.simiacryptus.jopenai.util.JsonUtil +import com.simiacryptus.skyenet.core.platform.ApplicationServices import com.simiacryptus.skyenet.core.platform.Session import com.simiacryptus.skyenet.core.platform.StorageInterface import com.simiacryptus.skyenet.core.platform.StorageInterface.Companion.validateSessionId @@ -9,7 +10,6 @@ import java.io.File import java.text.SimpleDateFormat import java.util.* - open class DataStorage( private val dataDir: File ) : StorageInterface { @@ -19,7 +19,10 @@ open class DataStorage( session: Session ): LinkedHashMap { validateSessionId(session) - val messageDir = SYS_DIR.resolve("${if (session.isGlobal()) "global" else user}/$session/messages/").apply { mkdirs() } + log.debug("Fetching messages for session: ${session.sessionId}, user: ${user?.email}") + val messageDir = + ApplicationServices.dataStorageRoot.resolve("${if (session.isGlobal()) "global" else user}/$session/messages/") + .apply { mkdirs() } val messages = LinkedHashMap() getMessageIds(user, session).forEach { messageId -> val file = File(messageDir, "$messageId.json") @@ -28,7 +31,7 @@ open class DataStorage( messages[messageId] = message } } - //log.debug("Loaded {} messages for {}", messages.size, session) + log.debug("Loaded ${messages.size} messages for session: ${session.sessionId}") return messages } @@ -40,6 +43,7 @@ open class DataStorage( return sessionPaths[session]!! } validateSessionId(session) + log.debug("Getting session directory for session: ${session.sessionId}, user: ${user?.email}") val parts = session.sessionId.split("-") return when (parts.size) { 3 -> { @@ -49,17 +53,15 @@ open class DataStorage( else -> throw IllegalArgumentException("Invalid session ID: $session") } val dateDir = File(root, parts[1]) - //log.debug("Date Dir for {}: {}", session, dateDir.absolutePath) val sessionDir = File(dateDir, parts[2]) - //log.debug("Instance Dir for {}: {}", session, sessionDir.absolutePath) + log.debug("Session directory for session: ${session.sessionId} is ${sessionDir.absolutePath}") sessionDir } 2 -> { val dateDir = File(dataDir, parts[0]) - //log.debug("Date Dir for {}: {}", session, dateDir.absolutePath) val sessionDir = File(dateDir, parts[1]) - //log.debug("Instance Dir for {}: {}", session, sessionDir.absolutePath) + log.debug("Session directory for session: ${session.sessionId} is ${sessionDir.absolutePath}") sessionDir } @@ -74,6 +76,7 @@ open class DataStorage( session: Session ): String { validateSessionId(session) + log.debug("Fetching session name for session: ${session.sessionId}, user: ${user?.email}") val sessionDir = getSessionDir(user, session) val settings = run { val settingsFile = File(sessionDir, "settings.json") @@ -86,10 +89,10 @@ open class DataStorage( messageFiles(session, user).entries.minByOrNull { it.key.lastModified() }?.value return if (null != userMessage) { setJson(sessionDir, "settings.json", settings.plus("name" to userMessage)) - //log.debug("Session {}: {}", session, userMessage) + log.debug("Session name for session: ${session.sessionId} is $userMessage") userMessage } else { - //log.debug("Session {}: No messages", session) + log.debug("Session ${session.sessionId} has no messages") session.sessionId } } @@ -99,9 +102,13 @@ open class DataStorage( session: Session ): List { validateSessionId(session) + log.debug("Fetching message IDs for session: ${session.sessionId}, user: ${user?.email}") val sessionDir = getSessionDir(user, session) val settings = run { - val settingsFile = File(SYS_DIR, "${if (session.isGlobal()) "global" else user}/$session/internal.json") + val settingsFile = File( + ApplicationServices.dataStorageRoot, + "${if (session.isGlobal()) "global" else user}/$session/internal.json" + ) if (!settingsFile.exists()) null else { JsonUtil.objectMapper().readValue(settingsFile, Map::class.java) as Map<*, *> } @@ -109,7 +116,12 @@ open class DataStorage( if (settings.containsKey("ids")) return settings["ids"].toString().split(",").toList() val ids = messageFiles(session, user).entries.sortedBy { it.key.lastModified() } .map { it.key.nameWithoutExtension }.toList() - setJson(SYS_DIR, "${if (session.isGlobal()) "global" else user}/$session/internal.json", settings.plus("ids" to ids.joinToString(","))) + setJson( + ApplicationServices.dataStorageRoot, + "${if (session.isGlobal()) "global" else user}/$session/internal.json", + settings.plus("ids" to ids.joinToString(",")) + ) + log.debug("Message IDs for session: ${session.sessionId} are $ids") return ids } @@ -119,13 +131,21 @@ open class DataStorage( ids: List ) { validateSessionId(session) + log.debug("Setting message IDs for session: ${session.sessionId}, user: ${user?.email} to $ids") val settings = run { - val settingsFile = File(SYS_DIR, "${if (session.isGlobal()) "global" else user}/$session/internal.json") + val settingsFile = File( + ApplicationServices.dataStorageRoot, + "${if (session.isGlobal()) "global" else user}/$session/internal.json" + ) if (!settingsFile.exists()) null else { JsonUtil.objectMapper().readValue(settingsFile, Map::class.java) as Map<*, *> } } ?: mapOf() - setJson(SYS_DIR, "${if (session.isGlobal()) "global" else user}/$session/internal.json", settings.plus("ids" to ids.joinToString(","))) + setJson( + ApplicationServices.dataStorageRoot, + "${if (session.isGlobal()) "global" else user}/$session/internal.json", + settings.plus("ids" to ids.joinToString(",")) + ) } override fun getSessionTime( @@ -133,7 +153,9 @@ open class DataStorage( session: Session ): Date? { validateSessionId(session) - val settingsFile = SYS_DIR.resolve("${if (session.isGlobal()) "global" else user}/$session/internal.json") + log.debug("Fetching session time for session: ${session.sessionId}, user: ${user?.email}") + val settingsFile = + ApplicationServices.dataStorageRoot.resolve("${if (session.isGlobal()) "global" else user}/$session/internal.json") val settings = run { if (!settingsFile.exists()) null else { JsonUtil.objectMapper().readValue(settingsFile, Map::class.java) as Map<*, *> @@ -145,10 +167,15 @@ open class DataStorage( val file = messageFiles.entries.minByOrNull { it.key.lastModified() }?.key return if (null != file) { val date = Date(file.lastModified()) - setJson(SYS_DIR, "${if (session.isGlobal()) "global" else user}/$session/internal.json", settings.plus("time" to dateFormat.format(date))) + setJson( + ApplicationServices.dataStorageRoot, + "${if (session.isGlobal()) "global" else user}/$session/internal.json", + settings.plus("time" to dateFormat.format(date)) + ) + log.debug("Session time for session: ${session.sessionId} is $date") date } else { - //log.debug("Session {}: No messages", session) + log.debug("Session ${session.sessionId} has no messages") null } } @@ -157,18 +184,19 @@ open class DataStorage( session: Session, user: User?, ) = - SYS_DIR.resolve("${if (session.isGlobal()) "global" else user}/$session/messages").apply { mkdirs() }.listFiles() + ApplicationServices.dataStorageRoot.resolve("${if (session.isGlobal()) "global" else user}/$session/messages") + .apply { mkdirs() }.listFiles() ?.filter { file -> file.isFile } ?.map { messageFile -> val fileText = messageFile.readText() val split = fileText.split("

") if (split.size < 2) { - //log.debug("Session {}: No messages", session) + log.debug("Session ${session.sessionId} has no messages in file ${messageFile.name}") messageFile to "" } else { val stringList = split[1].split("

") if (stringList.isEmpty()) { - //log.debug("Session {}: No messages", session) + log.debug("Session ${session.sessionId} has no messages in file ${messageFile.name}") messageFile to "" } else { messageFile to stringList.first() @@ -179,9 +207,23 @@ open class DataStorage( override fun listSessions( user: User? ): List { + log.debug("Listing sessions for user: ${user?.email}") val globalSessions = listSessions(dataDir) val userSessions = if (user == null) listOf() else listSessions(userRoot(user)) - return globalSessions.map { Session("G-$it") } + userSessions.map { Session("U-$it") } + log.debug("Found ${globalSessions.size} global sessions and ${userSessions.size} user sessions for user: ${user?.email}") + return ((globalSessions.map { + try { + Session("G-$it") + } catch (e: Exception) { + null + } + }).toList() + (userSessions.map { + try { + Session("U-$it") + } catch (e: Exception) { + null + } + }).toList()).filterNotNull() } override fun setJson( @@ -192,6 +234,7 @@ open class DataStorage( ) = setJson(getSessionDir(user, session), filename, settings) private fun setJson(sessionDir: File, filename: String, settings: T): T { + log.debug("Setting JSON for session directory: ${sessionDir.absolutePath}, filename: $filename") val settingsFile = File(sessionDir, filename) settingsFile.parentFile.mkdirs() JsonUtil.objectMapper().writeValue(settingsFile, settings) @@ -205,12 +248,14 @@ open class DataStorage( value: String ) { validateSessionId(session) - val file = SYS_DIR.resolve("${if (session.isGlobal()) "global" else user}/$session/messages/$messageId.json").apply { parentFile.mkdirs() } + log.debug("Updating message for session: ${session.sessionId}, messageId: $messageId, user: ${user?.email}") + val file = + ApplicationServices.dataStorageRoot.resolve("${if (session.isGlobal()) "global" else user}/$session/messages/$messageId.json") + .apply { parentFile.mkdirs() } if (!file.exists()) { file.parentFile.mkdirs() addMessageID(user, session, messageId) } - //log.debug("Updating message for {} / {}: {}", session, messageId, file.absolutePath) JsonUtil.objectMapper().writeValue(file, value) } @@ -220,11 +265,13 @@ open class DataStorage( messageId: String ) { synchronized(this) { + log.debug("Adding message ID for session: ${session.sessionId}, messageId: $messageId, user: ${user?.email}") setMessageIds(user, session, getMessageIds(user, session) + messageId) } } override fun listSessions(dir: File): List { + log.debug("Listing sessions in directory: ${dir.absolutePath}") val files = dir.listFiles()?.flatMap { it.listFiles()?.toList() ?: listOf() }?.filter { sessionDir -> val operationDir = File(sessionDir, "messages") if (!operationDir.exists()) false else { @@ -237,7 +284,7 @@ open class DataStorage( (listFiles?.size ?: 0) > 0 } }?.sortedBy { it.lastModified() } ?: listOf() - //log.debug("Sessions: {}", files.map { it.parentFile.name + "-" + it.name }) + log.debug("Found ${files.size} sessions in directory: ${dir.absolutePath}") return files.map { it.parentFile.name + "-" + it.name } } @@ -252,15 +299,15 @@ open class DataStorage( override fun deleteSession(user: User?, session: Session) { validateSessionId(session) + log.debug("Deleting session: ${session.sessionId}, user: ${user?.email}") val sessionDir = getSessionDir(user, session) sessionDir.deleteRecursively() } companion object { - var SYS_DIR = File(".sys" + File.separator) + val log = org.slf4j.LoggerFactory.getLogger(DataStorage::class.java) val sessionPaths = mutableMapOf() } -} - +} \ No newline at end of file diff --git a/core/src/main/kotlin/com/simiacryptus/skyenet/core/platform/file/UserSettingsManager.kt b/core/src/main/kotlin/com/simiacryptus/skyenet/core/platform/file/UserSettingsManager.kt index d7909ff4..e54c7feb 100644 --- a/core/src/main/kotlin/com/simiacryptus/skyenet/core/platform/file/UserSettingsManager.kt +++ b/core/src/main/kotlin/com/simiacryptus/skyenet/core/platform/file/UserSettingsManager.kt @@ -1,6 +1,7 @@ package com.simiacryptus.skyenet.core.platform.file import com.simiacryptus.jopenai.util.JsonUtil +import com.simiacryptus.skyenet.core.platform.ApplicationServices import com.simiacryptus.skyenet.core.platform.User import com.simiacryptus.skyenet.core.platform.UserSettingsInterface import com.simiacryptus.skyenet.core.platform.UserSettingsInterface.UserSettings @@ -9,7 +10,7 @@ import java.io.File open class UserSettingsManager : UserSettingsInterface { private val userSettings = HashMap() - private val userConfigDirectory = File(".skyenet/users") + private val userConfigDirectory by lazy { ApplicationServices.dataStorageRoot.resolve("users").apply { mkdirs() } } override fun getUserSettings(user: User): UserSettings { return userSettings.getOrPut(user) { diff --git a/gradle.properties b/gradle.properties index 4428ca29..412b08af 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,4 +1,4 @@ # Gradle Releases -> https://github.com/gradle/gradle/releases libraryGroup = com.simiacryptus.skyenet -libraryVersion = 1.0.71 +libraryVersion = 1.0.72 gradleVersion = 7.6.1 diff --git a/webui/src/main/kotlin/com/simiacryptus/skyenet/webui/application/ApplicationDirectory.kt b/webui/src/main/kotlin/com/simiacryptus/skyenet/webui/application/ApplicationDirectory.kt index 5f8b6e98..b3cc5f01 100644 --- a/webui/src/main/kotlin/com/simiacryptus/skyenet/webui/application/ApplicationDirectory.kt +++ b/webui/src/main/kotlin/com/simiacryptus/skyenet/webui/application/ApplicationDirectory.kt @@ -9,6 +9,7 @@ import com.simiacryptus.skyenet.webui.chat.ChatServer import com.simiacryptus.skyenet.webui.servlet.* import com.simiacryptus.skyenet.webui.util.Selenium2S3 import jakarta.servlet.DispatcherType +import jakarta.servlet.MultipartConfigElement import jakarta.servlet.Servlet import jakarta.servlet.http.HttpServlet import org.eclipse.jetty.server.* @@ -217,7 +218,9 @@ abstract class ApplicationDirectory( log.debug("New WebAppContext created for servlet at path: $path") context.resourceBase = "application" context.welcomeFiles = arrayOf("index.html") - context.addServlet(ServletHolder(servlet), "/") + val servletHolder = ServletHolder(servlet) + servletHolder.getRegistration().setMultipartConfig(MultipartConfigElement("./tmp")); + context.addServlet(servletHolder, "/") return context } diff --git a/webui/src/main/kotlin/com/simiacryptus/skyenet/webui/application/ApplicationServer.kt b/webui/src/main/kotlin/com/simiacryptus/skyenet/webui/application/ApplicationServer.kt index 2965523b..60902e12 100644 --- a/webui/src/main/kotlin/com/simiacryptus/skyenet/webui/application/ApplicationServer.kt +++ b/webui/src/main/kotlin/com/simiacryptus/skyenet/webui/application/ApplicationServer.kt @@ -2,15 +2,11 @@ package com.simiacryptus.skyenet.webui.application import com.simiacryptus.jopenai.API import com.simiacryptus.jopenai.util.JsonUtil +import com.simiacryptus.skyenet.core.platform.* import com.simiacryptus.skyenet.core.platform.ApplicationServices.authenticationManager import com.simiacryptus.skyenet.core.platform.ApplicationServices.authorizationManager import com.simiacryptus.skyenet.core.platform.ApplicationServices.dataStorageFactory -import com.simiacryptus.skyenet.core.platform.AuthenticationInterface import com.simiacryptus.skyenet.core.platform.AuthorizationInterface.OperationType -import com.simiacryptus.skyenet.core.platform.Session -import com.simiacryptus.skyenet.core.platform.StorageInterface -import com.simiacryptus.skyenet.core.platform.User -import com.simiacryptus.skyenet.core.platform.file.DataStorage.Companion.SYS_DIR import com.simiacryptus.skyenet.webui.chat.ChatServer import com.simiacryptus.skyenet.webui.servlet.* import com.simiacryptus.skyenet.webui.session.SocketManager @@ -25,7 +21,7 @@ abstract class ApplicationServer( final override val applicationName: String, val path: String, resourceBase: String = "application", - open val root: File = File(File(".skyenet"), applicationName), + open val root: File = ApplicationServices.dataStorageRoot, val showMenubar: Boolean = true, ) : ChatServer(resourceBase) { @@ -48,7 +44,7 @@ abstract class ApplicationServer( protected open val userInfo by lazy { ServletHolder("userInfo", UserInfoServlet()) } protected open val usageServlet by lazy { ServletHolder("usage", UsageServlet()) } protected open val fileZip by lazy { ServletHolder("fileZip", ZipServlet(dataStorage)) } - protected open val fileIndex by lazy { ServletHolder("fileIndex", FileServlet(dataStorage)) } + protected open val fileIndex by lazy { ServletHolder("fileIndex", SessionFileServlet(dataStorage)) } protected open val sessionSettingsServlet by lazy { ServletHolder("settings", SessionSettingsServlet(this)) } protected open val sessionShareServlet by lazy { ServletHolder("share", SessionShareServlet(this)) } protected open val sessionThreadsServlet by lazy { ServletHolder("threads", SessionThreadsServlet(this)) } @@ -94,8 +90,7 @@ abstract class ApplicationServer( userId: User?, @Suppress("UNCHECKED_CAST") clazz: Class = settingsClass as Class ): T? { - val settingsFile = SYS_DIR.resolve("${if (session.isGlobal()) "global" else userId}/$session/settings.json") - .apply { parentFile.mkdirs() } + val settingsFile = getSettingsFile(session, userId) var settings: T? = if(settingsFile.exists()) JsonUtil.fromJson(settingsFile.readText(), clazz) else null if (null == settings) { val initSettings = initSettings(session) @@ -109,6 +104,16 @@ abstract class ApplicationServer( return settings } + fun getSettingsFile( + session: Session, + userId: User? + ): File { + val settingsFile = + ApplicationServices.dataStorageRoot.resolve("${if (session.isGlobal()) "global" else userId}/$session/settings.json") + .apply { parentFile.mkdirs() } + return settingsFile + } + protected open fun sessionsServlet(path: String) = ServletHolder("sessionList", SessionListServlet(this.dataStorage, path, this)) diff --git a/webui/src/main/kotlin/com/simiacryptus/skyenet/webui/servlet/ApiKeyServlet.kt b/webui/src/main/kotlin/com/simiacryptus/skyenet/webui/servlet/ApiKeyServlet.kt index 9a2f688a..f98f74bc 100644 --- a/webui/src/main/kotlin/com/simiacryptus/skyenet/webui/servlet/ApiKeyServlet.kt +++ b/webui/src/main/kotlin/com/simiacryptus/skyenet/webui/servlet/ApiKeyServlet.kt @@ -299,10 +299,7 @@ class ApiKeyServlet : HttpServlet() { companion object { private val userRoot by lazy { - File( - File(ApplicationServices.dataStorageRoot, ".skyenet"), - "apiKeys" - ).apply { mkdirs() } + ApplicationServices.dataStorageRoot.resolve("apiKeys").apply { mkdirs() } } val apiKeyRecords by lazy { diff --git a/webui/src/main/kotlin/com/simiacryptus/skyenet/webui/servlet/FileServlet.kt b/webui/src/main/kotlin/com/simiacryptus/skyenet/webui/servlet/FileServlet.kt index 3a3b2237..061a4826 100644 --- a/webui/src/main/kotlin/com/simiacryptus/skyenet/webui/servlet/FileServlet.kt +++ b/webui/src/main/kotlin/com/simiacryptus/skyenet/webui/servlet/FileServlet.kt @@ -2,12 +2,9 @@ package com.simiacryptus.skyenet.webui.servlet import com.google.common.cache.CacheBuilder import com.google.common.cache.CacheLoader +import com.google.common.cache.LoadingCache import com.google.common.cache.RemovalListener -import com.simiacryptus.skyenet.core.platform.ApplicationServices -import com.simiacryptus.skyenet.core.platform.Session -import com.simiacryptus.skyenet.core.platform.StorageInterface import com.simiacryptus.skyenet.webui.application.ApplicationServer -import com.simiacryptus.skyenet.webui.application.ApplicationServer.Companion.getCookie import jakarta.servlet.WriteListener import jakarta.servlet.http.HttpServlet import jakarta.servlet.http.HttpServletRequest @@ -19,30 +16,41 @@ import java.nio.MappedByteBuffer import java.nio.channels.FileChannel import java.nio.file.StandardOpenOption -class FileServlet(val dataStorage: StorageInterface) : HttpServlet() { +abstract class FileServlet() : HttpServlet() { + + abstract fun getDir( + req: HttpServletRequest, + ) : File override fun doGet(req: HttpServletRequest, resp: HttpServletResponse) { - val pathSegments = parsePath(req.pathInfo ?: "/") - val session = Session(pathSegments.first()) - val sessionDir = - dataStorage.getSessionDir(ApplicationServices.authenticationManager.getUser(req.getCookie()), session) - val file = File(sessionDir, pathSegments.drop(1).joinToString("/")) + log.info("Received GET request for path: ${req.pathInfo ?: req.servletPath}") + val pathSegments = parsePath(req.pathInfo ?: req.servletPath ?: "/") + val dir = getDir(req) + log.info("Serving directory: ${dir.absolutePath}") + val file = getFile(dir, pathSegments, req) + log.info("Resolved file path: ${file.absolutePath}") + when { !file.exists() -> { + log.warn("File not found: ${file.absolutePath}") resp.status = HttpServletResponse.SC_NOT_FOUND resp.writer.write("File not found") } file.isFile -> { + log.info("File found: ${file.absolutePath}") var channel = channelCache.get(file) while (!channel.isOpen) { + log.warn("FileChannel is not open, refreshing cache for file: ${file.absolutePath}") channelCache.refresh(file) channel = channelCache.get(file) } try { if (channel.size() > 1024 * 1024 * 1) { + log.info("File is large, using writeLarge method for file: ${file.absolutePath}") writeLarge(channel, resp, file, req) } else { + log.info("File is small, using writeSmall method for file: ${file.absolutePath}") writeSmall(channel, resp, file, req) } } finally { @@ -51,10 +59,12 @@ class FileServlet(val dataStorage: StorageInterface) : HttpServlet() { } req.pathInfo?.endsWith("/") == false -> { + log.info("Redirecting to directory path: ${req.requestURI + "/"}") resp.sendRedirect(req.requestURI + "/") } else -> { + log.info("Listing directory contents for: ${file.absolutePath}") resp.contentType = "text/html" resp.status = HttpServletResponse.SC_OK val files = file.listFiles() @@ -71,12 +81,22 @@ class FileServlet(val dataStorage: StorageInterface) : HttpServlet() { ?.joinToString("
\n") { """${it.name}""" } ?: "" - resp.writer.write(directoryHTML(req, session, pathSegments.drop(1).joinToString("/"), folders, files)) + resp.writer.write( + directoryHTML( + getZipLink(req, pathSegments.drop(1).joinToString("/")), + folders, + files + ).trimMargin() + ) } } } + open fun getFile(dir: File, pathSegments: List, req: HttpServletRequest) = + File(dir, pathSegments.drop(1).joinToString("/")) + private fun writeSmall(channel: FileChannel, resp: HttpServletResponse, file: File, req: HttpServletRequest) { + log.info("Writing small file: ${file.absolutePath}") resp.contentType = ApplicationServer.getMimeType(file.name) resp.status = HttpServletResponse.SC_OK val async = req.startAsync() @@ -89,6 +109,7 @@ class FileServlet(val dataStorage: StorageInterface) : HttpServlet() { byteBuffer.clear() val readBytes = channel.read(byteBuffer) if (readBytes == -1) { + log.info("Completed writing small file: ${file.absolutePath}") async.complete() channelCache.put(file, channel) return @@ -98,7 +119,7 @@ class FileServlet(val dataStorage: StorageInterface) : HttpServlet() { } override fun onError(throwable: Throwable) { - log.warn("Error writing file", throwable) + log.error("Error writing small file: ${file.absolutePath}", throwable) channelCache.put(file, channel) } }) @@ -111,6 +132,7 @@ class FileServlet(val dataStorage: StorageInterface) : HttpServlet() { file: File, req: HttpServletRequest ) { + log.info("Writing large file: ${file.absolutePath}") val mappedByteBuffer: MappedByteBuffer = channel.map(FileChannel.MapMode.READ_ONLY, 0, channel.size()) resp.contentType = ApplicationServer.getMimeType(file.name) resp.status = HttpServletResponse.SC_OK @@ -126,6 +148,7 @@ class FileServlet(val dataStorage: StorageInterface) : HttpServlet() { val end = mappedByteBuffer.position() val readBytes = end - start if (readBytes == 0) { + log.info("Completed writing large file: ${file.absolutePath}") async.complete() channelCache.put(file, channel) return @@ -135,93 +158,92 @@ class FileServlet(val dataStorage: StorageInterface) : HttpServlet() { } override fun onError(throwable: Throwable) { - log.warn("Error writing file", throwable) + log.error("Error writing large file: ${file.absolutePath}", throwable) channelCache.put(file, channel) } }) } } - private fun directoryHTML( + open fun getZipLink( req: HttpServletRequest, - session: Session, - filePath: String, - folders: String, - files: String - ) = """ - | - | - |Files - | - | - | - |

Archive

- |ZIP - |

Folders

- |
- |$folders - |
- |

Files

- |
- |$files - |
- | - | - """.trimMargin() + filePath: String + ) : String = "" + + private fun directoryHTML(zipLink: String, folders: String, files: String) = """ + | + | + |Files + | + | + | + |

Archive

+ |${if(zipLink.isNullOrBlank()) "" else """ZIP"""} + |

Folders

+ |
+ |$folders + |
+ |

Files

+ |
+ |$files + |
+ | + | + """ companion object { val log = LoggerFactory.getLogger(FileServlet::class.java) @@ -246,7 +268,7 @@ class FileServlet(val dataStorage: StorageInterface) : HttpServlet() { return pathSegments } - val channelCache = CacheBuilder + val channelCache: LoadingCache = CacheBuilder .newBuilder().maximumSize(100) .expireAfterAccess(10, java.util.concurrent.TimeUnit.SECONDS) .removalListener(RemovalListener { notification -> @@ -264,6 +286,7 @@ class FileServlet(val dataStorage: StorageInterface) : HttpServlet() { } }).build(object : CacheLoader() { override fun load(key: File): FileChannel { + log.info("Opening FileChannel for file: ${key.absolutePath}") return FileChannel.open(key.toPath(), StandardOpenOption.READ) } }) diff --git a/webui/src/main/kotlin/com/simiacryptus/skyenet/webui/servlet/SessionFileServlet.kt b/webui/src/main/kotlin/com/simiacryptus/skyenet/webui/servlet/SessionFileServlet.kt new file mode 100644 index 00000000..20d04119 --- /dev/null +++ b/webui/src/main/kotlin/com/simiacryptus/skyenet/webui/servlet/SessionFileServlet.kt @@ -0,0 +1,24 @@ +package com.simiacryptus.skyenet.webui.servlet + +import com.simiacryptus.skyenet.core.platform.ApplicationServices +import com.simiacryptus.skyenet.core.platform.Session +import com.simiacryptus.skyenet.core.platform.StorageInterface +import com.simiacryptus.skyenet.webui.application.ApplicationServer.Companion.getCookie +import jakarta.servlet.http.HttpServletRequest +import java.io.File + +class SessionFileServlet(val dataStorage: StorageInterface) : FileServlet() { + override fun getDir( + req: HttpServletRequest, + ): File { + val pathSegments = parsePath(req.pathInfo ?: "/") + val session = Session(pathSegments.first()) + return dataStorage.getSessionDir(ApplicationServices.authenticationManager.getUser(req.getCookie()), session) + } + + override fun getZipLink(req: HttpServletRequest, filePath: String): String { + val pathSegments = parsePath(req.pathInfo ?: "/") + val session = Session(pathSegments.first()) + return "${req.contextPath}/fileZip?session=$session&path=$filePath" + } +} \ No newline at end of file diff --git a/webui/src/main/kotlin/com/simiacryptus/skyenet/webui/servlet/SessionSettingsServlet.kt b/webui/src/main/kotlin/com/simiacryptus/skyenet/webui/servlet/SessionSettingsServlet.kt index 9fc872a7..0e925605 100644 --- a/webui/src/main/kotlin/com/simiacryptus/skyenet/webui/servlet/SessionSettingsServlet.kt +++ b/webui/src/main/kotlin/com/simiacryptus/skyenet/webui/servlet/SessionSettingsServlet.kt @@ -3,13 +3,11 @@ package com.simiacryptus.skyenet.webui.servlet import com.simiacryptus.jopenai.util.JsonUtil import com.simiacryptus.skyenet.core.platform.ApplicationServices.authenticationManager import com.simiacryptus.skyenet.core.platform.Session -import com.simiacryptus.skyenet.core.platform.file.DataStorage.Companion.SYS_DIR import com.simiacryptus.skyenet.webui.application.ApplicationServer import com.simiacryptus.skyenet.webui.application.ApplicationServer.Companion.getCookie import jakarta.servlet.http.HttpServlet import jakarta.servlet.http.HttpServletRequest import jakarta.servlet.http.HttpServletResponse -import java.io.File class SessionSettingsServlet( private val server: ApplicationServer, @@ -23,6 +21,11 @@ class SessionSettingsServlet( val user = authenticationManager.getUser(req.getCookie()) val settings = server.getSettings(session, user, settingsClass) val json = if (settings != null) JsonUtil.toJson(settings) else "" + if (req.parameterMap.containsKey("raw") && req.getParameter("raw") == "true") { + resp.contentType = "application/json" + resp.writer.write(json) + return + } //language=HTML resp.writer.write( """ @@ -55,11 +58,21 @@ class SessionSettingsServlet( resp.status = HttpServletResponse.SC_BAD_REQUEST resp.writer.write("Session ID is required") } else { - val session = Session(req.getParameter("sessionId")) - val settings = JsonUtil.fromJson(req.getParameter("settings"), settingsClass) - val user = authenticationManager.getUser(req.getCookie()) ?: "global" - SYS_DIR.resolve("$user/$session/settings.json").apply { parentFile.mkdirs() }.writeText(JsonUtil.toJson(settings)) - resp.sendRedirect("${req.contextPath}/#$session") + if (!req.parameterMap.containsKey("sessionId")) { + resp.status = HttpServletResponse.SC_BAD_REQUEST + resp.writer.write("Session ID is required") + } else { + val session = Session(req.getParameter("sessionId")) + val settings = if (req.parameterNames.toList().contains("settings")) { + JsonUtil.fromJson(req.getParameter("settings"), settingsClass) + } else { + JsonUtil.fromJson(req.reader.readText(), settingsClass) + } + val user = authenticationManager.getUser(req.getCookie()) + val settingsFile = server.getSettingsFile(session, user).apply { parentFile.mkdirs() } + settingsFile.writeText(JsonUtil.toJson(settings)) + resp.sendRedirect("${req.contextPath}/#$session") + } } } } \ No newline at end of file diff --git a/webui/src/main/kotlin/com/simiacryptus/skyenet/webui/servlet/SessionShareServlet.kt b/webui/src/main/kotlin/com/simiacryptus/skyenet/webui/servlet/SessionShareServlet.kt index 1d483e32..40dd650d 100644 --- a/webui/src/main/kotlin/com/simiacryptus/skyenet/webui/servlet/SessionShareServlet.kt +++ b/webui/src/main/kotlin/com/simiacryptus/skyenet/webui/servlet/SessionShareServlet.kt @@ -5,11 +5,11 @@ import com.simiacryptus.skyenet.core.platform.ApplicationServices import com.simiacryptus.skyenet.core.platform.ApplicationServices.authenticationManager import com.simiacryptus.skyenet.core.platform.ApplicationServices.authorizationManager import com.simiacryptus.skyenet.core.platform.ApplicationServices.cloud +import com.simiacryptus.skyenet.core.platform.ApplicationServices.dataStorageRoot import com.simiacryptus.skyenet.core.platform.AuthorizationInterface.OperationType import com.simiacryptus.skyenet.core.platform.StorageInterface import com.simiacryptus.skyenet.core.platform.StorageInterface.Companion.long64 import com.simiacryptus.skyenet.core.platform.User -import com.simiacryptus.skyenet.core.platform.file.DataStorage.Companion.SYS_DIR import com.simiacryptus.skyenet.core.util.Selenium import com.simiacryptus.skyenet.webui.application.ApplicationServer import com.simiacryptus.skyenet.webui.application.ApplicationServer.Companion.getCookie @@ -46,10 +46,12 @@ class SessionShareServlet( require(acceptHost(user, host)) { "Invalid url: $url" } - val storageInterface = ApplicationServices.dataStorageFactory.invoke(File(File(".skyenet"), appName)) + val storageInterface = ApplicationServices.dataStorageFactory.invoke(File(dataStorageRoot, appName)) val session = StorageInterface.parseSessionID(sessionID) val pool = ApplicationServices.clientManager.getPool(session, user, server.dataStorage) - val json = JsonUtil.fromJson>(SYS_DIR.resolve("${if (session.isGlobal()) "global" else user}/$session/info.json").apply { parentFile.mkdirs() }.readText(), typeOf>().javaType) + val json = JsonUtil.fromJson>( + dataStorageRoot.resolve("${if (session.isGlobal()) "global" else user}/$session/info.json") + .apply { parentFile.mkdirs() }.readText(), typeOf>().javaType) val sessionSettings = (json as? Map)?.toMutableMap() ?: mutableMapOf() val previousShare = sessionSettings["shareId"] when { @@ -90,7 +92,8 @@ class SessionShareServlet( try { log.info("Generating shareId: $shareId") sessionSettings["shareId"] = shareId - SYS_DIR.resolve("${if (session.isGlobal()) "global" else user}/$session/info.json").apply { parentFile.mkdirs() }.writeText(JsonUtil.toJson(sessionSettings)) + dataStorageRoot.resolve("${if (session.isGlobal()) "global" else user}/$session/info.json") + .apply { parentFile.mkdirs() }.writeText(JsonUtil.toJson(sessionSettings)) // val selenium2S3 = Selenium2S3( // pool = pool, // cookies = cookies, diff --git a/webui/src/main/kotlin/com/simiacryptus/skyenet/webui/servlet/WelcomeServlet.kt b/webui/src/main/kotlin/com/simiacryptus/skyenet/webui/servlet/WelcomeServlet.kt index 7f07c3ba..dc6bb26f 100644 --- a/webui/src/main/kotlin/com/simiacryptus/skyenet/webui/servlet/WelcomeServlet.kt +++ b/webui/src/main/kotlin/com/simiacryptus/skyenet/webui/servlet/WelcomeServlet.kt @@ -63,8 +63,12 @@ open class WelcomeServlet(private val parent: ApplicationDirectory) : SimiaCryptus Skyenet Apps - + + + + +