diff --git a/core/build.gradle.kts b/core/build.gradle.kts index 813af374..b3c4d640 100644 --- a/core/build.gradle.kts +++ b/core/build.gradle.kts @@ -33,7 +33,7 @@ val hsqldb_version = "2.7.2" dependencies { - implementation(group = "com.simiacryptus", name = "jo-penai", version = "1.1.10") + implementation(group = "com.simiacryptus", name = "jo-penai", version = "1.1.11") implementation(group = "org.hsqldb", name = "hsqldb", version = hsqldb_version) implementation("org.apache.commons:commons-text:1.11.0") diff --git a/core/src/main/kotlin/com/simiacryptus/skyenet/core/platform/AwsPlatform.kt b/core/src/main/kotlin/com/simiacryptus/skyenet/core/platform/AwsPlatform.kt index e728f385..bd778433 100644 --- a/core/src/main/kotlin/com/simiacryptus/skyenet/core/platform/AwsPlatform.kt +++ b/core/src/main/kotlin/com/simiacryptus/skyenet/core/platform/AwsPlatform.kt @@ -18,13 +18,17 @@ open class AwsPlatform( override val shareBase: String = System.getProperty("share_base", "https://share.simiacrypt.us"), private val region: Region? = Region.US_EAST_1 ) : CloudPlatformInterface { + private val log = LoggerFactory.getLogger(AwsPlatform::class.java) + protected open val kmsClient: KmsClient by lazy { + log.debug("Initializing KMS client for region: {}", Region.US_EAST_1) KmsClient.builder().region(Region.US_EAST_1) //.credentialsProvider(ProfileCredentialsProvider.create("data")) .build() } protected open val s3Client: S3Client by lazy { + log.debug("Initializing S3 client for region: {}", region) S3Client.builder() .region(region) .build() @@ -35,6 +39,7 @@ open class AwsPlatform( contentType: String, bytes: ByteArray ): String { + log.info("Uploading {} bytes to S3 path: {}", bytes.size, path) s3Client.putObject( PutObjectRequest.builder() .bucket(bucket).key(path.replace("/{2,}".toRegex(), "/").removePrefix("/")) @@ -42,6 +47,7 @@ open class AwsPlatform( .build(), RequestBody.fromBytes(bytes) ) + log.debug("Upload completed successfully") return "$shareBase/$path" } @@ -50,6 +56,7 @@ open class AwsPlatform( contentType: String, request: String ): String { + log.info("Uploading string content to S3 path: {}", path) s3Client.putObject( PutObjectRequest.builder() .bucket(bucket).key(path.replace("/{2,}".toRegex(), "/").removePrefix("/")) @@ -57,12 +64,14 @@ open class AwsPlatform( .build(), RequestBody.fromString(request) ) + log.debug("Upload completed successfully") return "$shareBase/$path" } - override fun encrypt(fileBytes: ByteArray, keyId: String): String? = - Base64.getEncoder().encodeToString( + override fun encrypt(fileBytes: ByteArray, keyId: String): String? { + log.info("Encrypting {} bytes using KMS key: {}", fileBytes.size, keyId) + val encryptedData = Base64.getEncoder().encodeToString( kmsClient.encrypt( EncryptRequest.builder() .keyId(keyId) @@ -70,18 +79,27 @@ open class AwsPlatform( .build() ).ciphertextBlob().asByteArray() ) + log.debug("Encryption completed successfully") + return encryptedData + } - override fun decrypt(encryptedData: ByteArray): String = String( - kmsClient.decrypt( - DecryptRequest.builder() - .ciphertextBlob(SdkBytes.fromByteArray(Base64.getDecoder().decode(encryptedData))) - .build() - ).plaintext().asByteArray(), StandardCharsets.UTF_8 - ) + override fun decrypt(encryptedData: ByteArray): String { + log.info("Decrypting {} bytes of data", encryptedData.size) + val decryptedData = String( + kmsClient.decrypt( + DecryptRequest.builder() + .ciphertextBlob(SdkBytes.fromByteArray(Base64.getDecoder().decode(encryptedData))) + .build() + ).plaintext().asByteArray(), StandardCharsets.UTF_8 + ) + log.debug("Decryption completed successfully") + return decryptedData + } companion object { val log = LoggerFactory.getLogger(AwsPlatform::class.java) fun get() = try { + log.info("Initializing AwsPlatform") AwsPlatform() } catch (e: Throwable) { log.warn("Error initializing AWS platform", e) diff --git a/core/src/main/kotlin/com/simiacryptus/skyenet/core/platform/file/AuthorizationManager.kt b/core/src/main/kotlin/com/simiacryptus/skyenet/core/platform/file/AuthorizationManager.kt index e999d810..85361131 100644 --- a/core/src/main/kotlin/com/simiacryptus/skyenet/core/platform/file/AuthorizationManager.kt +++ b/core/src/main/kotlin/com/simiacryptus/skyenet/core/platform/file/AuthorizationManager.kt @@ -11,21 +11,23 @@ open class AuthorizationManager : AuthorizationInterface { user: User?, operationType: AuthorizationInterface.OperationType, ) = try { + log.debug("Checking authorization for user: {}, operation: {}, application: {}", user, operationType, applicationClass) if (isUserAuthorized("/permissions/${operationType.name.lowercase(Locale.getDefault())}.txt", user)) { - log.debug("User {} authorized for {} globally", user, operationType) + log.info("User {} authorized for {} globally", user, operationType) true } else if (null != applicationClass) { val packagePath = applicationClass.`package`.name.replace('.', '/') val opName = operationType.name.lowercase(Locale.getDefault()) + log.debug("Checking application-specific authorization at path: /permissions/{}/{}.txt", packagePath, opName) if (isUserAuthorized("/permissions/$packagePath/$opName.txt", user)) { - log.debug("User {} authorized for {} on {}", user, operationType, applicationClass) + log.info("User {} authorized for {} on {}", user, operationType, applicationClass) true } else { - log.debug("User {} not authorized for {} on {}", user, operationType, applicationClass) + log.warn("User {} not authorized for {} on {}", user, operationType, applicationClass) false } } else { - log.debug("User {} not authorized for {} globally", user, operationType) + log.warn("User {} not authorized for {} globally", user, operationType) false } } catch (e: Exception) { @@ -33,24 +35,47 @@ open class AuthorizationManager : AuthorizationInterface { false } - private fun isUserAuthorized(permissionPath: String, user: User?) = - javaClass.getResourceAsStream(permissionPath)?.use { stream -> + private fun isUserAuthorized(permissionPath: String, user: User?): Boolean { + log.debug("Checking user authorization at path: {}", permissionPath) + return javaClass.getResourceAsStream(permissionPath)?.use { stream -> val lines = stream.bufferedReader().readLines() + log.trace("Permission file contents: {}", lines) lines.any { line -> matches(user, line) } - } ?: false + } ?: run { + log.warn("Permission file not found: {}", permissionPath) + false + } + } - open fun matches(user: User?, line: String) = when { - line.equals(user?.email, ignoreCase = true) -> true // Exact match - line.startsWith("@") && user?.email?.endsWith(line.substring(1)) == true -> true // Domain match - line == "." && user != null -> true // Any user - line == "*" -> true // Any user including anonymous - else -> false + open fun matches(user: User?, line: String): Boolean { + log.trace("Matching user {} against line: {}", user, line) + return when { + line.equals(user?.email, ignoreCase = true) -> { + log.debug("Exact match found for user: {}", user) + true + } + line.startsWith("@") && user?.email?.endsWith(line.substring(1)) == true -> { + log.debug("Domain match found for user: {}", user) + true + } + line == "." && user != null -> { + log.debug("Any authenticated user match for: {}", user) + true + } + line == "*" -> { + log.debug("Any user (including anonymous) match") + true + } + else -> { + log.trace("No match found for user: {} and line: {}", user, line) + false + } + } } companion object { private val log = org.slf4j.LoggerFactory.getLogger(AuthorizationManager::class.java) } - } \ No newline at end of file diff --git a/core/src/main/kotlin/com/simiacryptus/skyenet/core/platform/file/UserSettingsManager.kt b/core/src/main/kotlin/com/simiacryptus/skyenet/core/platform/file/UserSettingsManager.kt index 5dd0df58..a151a2fb 100644 --- a/core/src/main/kotlin/com/simiacryptus/skyenet/core/platform/file/UserSettingsManager.kt +++ b/core/src/main/kotlin/com/simiacryptus/skyenet/core/platform/file/UserSettingsManager.kt @@ -13,26 +13,33 @@ open class UserSettingsManager : UserSettingsInterface { private val userConfigDirectory by lazy { dataStorageRoot.resolve("users").apply { mkdirs() } } override fun getUserSettings(user: User): UserSettings { + log.debug("Retrieving user settings for user: {}", user) return userSettings.getOrPut(user) { val file = File(userConfigDirectory, "$user.json") if (file.exists()) { try { - log.info("Loading user settings for $user from $file") + log.info("Loading existing user settings for user: {} from file: {}", user, file) return@getOrPut JsonUtil.fromJson(file.readText(), UserSettings::class.java) } catch (e: Throwable) { - log.warn("Error loading user settings for $user from $file", e) + log.error("Failed to load user settings for user: {} from file: {}. Creating new settings.", user, file, e) } } - log.info("Creating new user settings for $user at $file", RuntimeException()) + log.info("User settings file not found for user: {}. Creating new settings at: {}", user, file) return@getOrPut UserSettings() } } override fun updateUserSettings(user: User, settings: UserSettings) { + log.debug("Updating user settings for user: {}", user) userSettings[user] = settings val file = File(userConfigDirectory, "$user.json") file.parentFile.mkdirs() - file.writeText(JsonUtil.toJson(settings)) + try { + file.writeText(JsonUtil.toJson(settings)) + log.info("Successfully updated user settings for user: {} at file: {}", user, file) + } catch (e: Exception) { + log.error("Failed to write user settings for user: {} to file: {}", user, file, e) + } } companion object { diff --git a/core/src/main/kotlin/com/simiacryptus/skyenet/core/platform/hsql/HSQLMetadataStorage.kt b/core/src/main/kotlin/com/simiacryptus/skyenet/core/platform/hsql/HSQLMetadataStorage.kt index 8940c26b..987fb84c 100644 --- a/core/src/main/kotlin/com/simiacryptus/skyenet/core/platform/hsql/HSQLMetadataStorage.kt +++ b/core/src/main/kotlin/com/simiacryptus/skyenet/core/platform/hsql/HSQLMetadataStorage.kt @@ -11,19 +11,19 @@ import java.sql.Timestamp import java.util.* class HSQLMetadataStorage(private val dbFile: File) : MetadataStorageInterface { - private val log = LoggerFactory.getLogger(HSQLMetadataStorage::class.java) + private val log = LoggerFactory.getLogger(javaClass) private val connection: Connection by lazy { log.info("Initializing HSQLMetadataStorage with database file: ${dbFile.absolutePath}") Class.forName("org.hsqldb.jdbc.JDBCDriver") val connection = DriverManager.getConnection("jdbc:hsqldb:file:${dbFile.absolutePath}/metadata;shutdown=true", "SA", "") - log.debug("Database connection established: $connection") + log.info("Database connection established successfully") createSchema(connection) connection } private fun createSchema(connection: Connection) { - log.info("Creating database schema if not exists") + log.debug("Attempting to create database schema if not exists") connection.createStatement().executeUpdate( """ CREATE TABLE IF NOT EXISTS metadata ( @@ -36,6 +36,7 @@ class HSQLMetadataStorage(private val dbFile: File) : MetadataStorageInterface { ) """ ) + log.info("Database schema creation completed") } override fun getSessionName(user: User?, session: Session): String { @@ -47,9 +48,10 @@ class HSQLMetadataStorage(private val dbFile: File) : MetadataStorageInterface { statement.setString(2, user?.email ?: "") val resultSet = statement.executeQuery() return if (resultSet.next()) { - resultSet.getString("value") + val name = resultSet.getString("value") + log.debug("Retrieved session name: $name for session: ${session.sessionId}") + name } else { - log.debug("Session ${session.sessionId} has no name") session.sessionId } } @@ -70,6 +72,7 @@ class HSQLMetadataStorage(private val dbFile: File) : MetadataStorageInterface { statement.setString(4, name) statement.setTimestamp(5, Timestamp(System.currentTimeMillis())) statement.executeUpdate() + log.info("Session name set successfully for session: ${session.sessionId}") } override fun getMessageIds(user: User?, session: Session): List { @@ -81,8 +84,11 @@ class HSQLMetadataStorage(private val dbFile: File) : MetadataStorageInterface { statement.setString(2, user?.email ?: "") val resultSet = statement.executeQuery() return if (resultSet.next()) { - resultSet.getString("value").split(",") + val ids = resultSet.getString("value").split(",") + log.debug("Retrieved ${ids.size} message IDs for session: ${session.sessionId}") + ids } else { + log.debug("No message IDs found for session: ${session.sessionId}") emptyList() } } @@ -103,6 +109,7 @@ class HSQLMetadataStorage(private val dbFile: File) : MetadataStorageInterface { statement.setString(4, ids.joinToString(",")) statement.setTimestamp(5, Timestamp(System.currentTimeMillis())) statement.executeUpdate() + log.info("Set ${ids.size} message IDs for session: ${session.sessionId}") } override fun getSessionTime(user: User?, session: Session): Date? { @@ -114,14 +121,16 @@ class HSQLMetadataStorage(private val dbFile: File) : MetadataStorageInterface { statement.setString(2, user?.email ?: "") val resultSet = statement.executeQuery() return if (resultSet.next()) { + val time = resultSet.getString("value") try { - Date(resultSet.getString("value").toLong()) + Date(time.toLong()).also { + log.debug("Retrieved session time: $it for session: ${session.sessionId}") + } } catch (e: NumberFormatException) { - log.warn("Invalid session time value, falling back to timestamp") + log.warn("Invalid session time value: $time, falling back to timestamp for session: ${session.sessionId}") resultSet.getTimestamp("timestamp") } } else { - log.debug("No session time found, returning current time") Date() } } @@ -142,6 +151,7 @@ class HSQLMetadataStorage(private val dbFile: File) : MetadataStorageInterface { statement.setString(4, time.time.toString()) statement.setTimestamp(5, Timestamp(time.time)) statement.executeUpdate() + log.info("Session time set to $time for session: ${session.sessionId}") } override fun listSessions(path: String): List { @@ -155,7 +165,7 @@ class HSQLMetadataStorage(private val dbFile: File) : MetadataStorageInterface { while (resultSet.next()) { sessions.add(resultSet.getString("session_id")) } - log.debug("Found ${sessions.size} sessions for path: $path") + log.info("Found ${sessions.size} sessions for path: $path") return sessions } @@ -167,9 +177,8 @@ class HSQLMetadataStorage(private val dbFile: File) : MetadataStorageInterface { statement.setString(1, session.sessionId) statement.setString(2, user?.email ?: "") statement.executeUpdate() + log.info("Deleted session: ${session.sessionId} for user: ${user?.email ?: "anonymous"}") } - companion object { - private val log = LoggerFactory.getLogger(HSQLMetadataStorage::class.java) - } + } \ No newline at end of file diff --git a/core/src/main/kotlin/com/simiacryptus/skyenet/core/platform/test/MetadataStorageInterfaceTest.kt b/core/src/main/kotlin/com/simiacryptus/skyenet/core/platform/test/MetadataStorageInterfaceTest.kt index d299c464..58cf6414 100644 --- a/core/src/main/kotlin/com/simiacryptus/skyenet/core/platform/test/MetadataStorageInterfaceTest.kt +++ b/core/src/main/kotlin/com/simiacryptus/skyenet/core/platform/test/MetadataStorageInterfaceTest.kt @@ -6,121 +6,162 @@ import com.simiacryptus.skyenet.core.platform.model.User import org.junit.jupiter.api.Assertions.* import org.junit.jupiter.api.Test import java.util.* +import org.slf4j.LoggerFactory abstract class MetadataStorageInterfaceTest(val storage: MetadataStorageInterface) { + companion object { + private val log = LoggerFactory.getLogger(MetadataStorageInterfaceTest::class.java) + } + @Test fun testGetSessionName() { + log.info("Starting testGetSessionName") // Arrange val user = User(email = "test@example.com") val session = Session("G-20230101-1234") // Act + log.debug("Retrieving session name for user {} and session {}", user.email, session.sessionId) val sessionName = storage.getSessionName(user, session) // Assert + log.debug("Retrieved session name: {}", sessionName) assertNotNull(sessionName) assertTrue(sessionName is String) + log.info("Completed testGetSessionName successfully") } @Test fun testSetSessionName() { + log.info("Starting testSetSessionName") // Arrange val user = User(email = "test@example.com") val session = Session("G-20230101-1234") val newName = "Test Session" // Act + log.debug("Setting session name to '{}' for user {} and session {}", newName, user.email, session.sessionId) storage.setSessionName(user, session, newName) + log.debug("Retrieving session name for verification") val retrievedName = storage.getSessionName(user, session) // Assert + log.debug("Retrieved session name: {}", retrievedName) assertEquals(newName, retrievedName) + log.info("Completed testSetSessionName successfully") } @Test fun testGetMessageIds() { + log.info("Starting testGetMessageIds") // Arrange val user = User(email = "test@example.com") val session = Session("G-20230101-1234") // Act + log.debug("Retrieving message IDs for user {} and session {}", user.email, session.sessionId) val messageIds = storage.getMessageIds(user, session) // Assert + log.debug("Retrieved message IDs: {}", messageIds) assertNotNull(messageIds) assertTrue(messageIds is List<*>) + log.info("Completed testGetMessageIds successfully") } @Test fun testSetMessageIds() { + log.info("Starting testSetMessageIds") // Arrange val user = User(email = "test@example.com") val session = Session("G-20230101-1234") val newIds = listOf("msg001", "msg002", "msg003") // Act + log.debug("Setting message IDs {} for user {} and session {}", newIds, user.email, session.sessionId) storage.setMessageIds(user, session, newIds) + log.debug("Retrieving message IDs for verification") val retrievedIds = storage.getMessageIds(user, session) // Assert + log.debug("Retrieved message IDs: {}", retrievedIds) assertEquals(newIds, retrievedIds) + log.info("Completed testSetMessageIds successfully") } // @Test fun testGetSessionTime() { + log.info("Starting testGetSessionTime") // Arrange val user = User(email = "test@example.com") val session = Session("G-20230101-1234") // Act + log.debug("Retrieving session time for user {} and session {}", user.email, session.sessionId) val sessionTime = storage.getSessionTime(user, session) // Assert + log.debug("Retrieved session time: {}", sessionTime) assertNotNull(sessionTime) assertTrue(sessionTime is Date) + log.info("Completed testGetSessionTime successfully") } @Test fun testSetSessionTime() { + log.info("Starting testSetSessionTime") // Arrange val user = User(email = "test@example.com") val session = Session("G-20230101-1234") val newTime = Date() // Act + log.debug("Setting session time to {} for user {} and session {}", newTime, user.email, session.sessionId) storage.setSessionTime(user, session, newTime) + log.debug("Retrieving session time for verification") val retrievedTime = storage.getSessionTime(user, session) // Assert + log.debug("Retrieved session time: {}", retrievedTime) assertEquals(newTime.toString(), retrievedTime.toString()) + log.info("Completed testSetSessionTime successfully") } @Test fun testListSessions() { + log.info("Starting testListSessions") // Arrange val path = "" // Act + log.debug("Listing sessions for path: {}", path) val sessions = storage.listSessions(path) // Assert + log.debug("Retrieved sessions: {}", sessions) assertNotNull(sessions) assertTrue(sessions is List<*>) + log.info("Completed testListSessions successfully") } @Test fun testDeleteSession() { + log.info("Starting testDeleteSession") // Arrange val user = User(email = "test@example.com") val session = Session("G-20230101-1234") // Act and Assert try { + log.debug("Attempting to delete session {} for user {}", session.sessionId, user.email) storage.deleteSession(user, session) + log.info("Session deleted successfully") // If no exception is thrown, the test passes. } catch (e: Exception) { + log.error("Failed to delete session: {}", e.message, e) fail("Exception should not be thrown") } + log.info("Completed testDeleteSession successfully") } } \ No newline at end of file diff --git a/core/src/main/kotlin/com/simiacryptus/skyenet/core/platform/test/StorageInterfaceTest.kt b/core/src/main/kotlin/com/simiacryptus/skyenet/core/platform/test/StorageInterfaceTest.kt index 909b9e3f..5465cb05 100644 --- a/core/src/main/kotlin/com/simiacryptus/skyenet/core/platform/test/StorageInterfaceTest.kt +++ b/core/src/main/kotlin/com/simiacryptus/skyenet/core/platform/test/StorageInterfaceTest.kt @@ -12,95 +12,124 @@ import java.io.File import java.util.* abstract class StorageInterfaceTest(val storage: StorageInterface) { + companion object { + private val log = org.slf4j.LoggerFactory.getLogger(StorageInterfaceTest::class.java) + } @Test fun testGetJson() { + log.info("Starting testGetJson") // Arrange val user = User(email = "test@example.com") val session = Session("G-20230101-1234") val filename = "test.json" // Act + log.debug("Attempting to read JSON file: {}", filename) val settingsFile = File(storage.getSessionDir(user, session), filename) val result = if (!settingsFile.exists()) null else { JsonUtil.objectMapper().readValue(settingsFile, Any::class.java) as Any } // Assert + log.info("Asserting result is null for non-existing JSON file") Assertions.assertNull(result, "Expected null result for non-existing JSON file") + log.info("testGetJson completed successfully") } @Test fun testGetMessages() { + log.info("Starting testGetMessages") // Arrange val user = User(email = "test@example.com") val session = Session("G-20230101-1234") // Act + log.debug("Retrieving messages for user: {} and session: {}", user.email, session.sessionId) val messages = storage.getMessages(user, session) // Assert + log.info("Asserting messages type is LinkedHashMap") assertTrue(messages is LinkedHashMap<*, *>, "Expected LinkedHashMap type for messages") + log.info("testGetMessages completed successfully") } @Test fun testGetSessionDir() { + log.info("Starting testGetSessionDir") // Arrange val user = User(email = "test@example.com") val session = Session("G-20230101-1234") // Act + log.debug("Getting session directory for user: {} and session: {}", user.email, session.sessionId) val sessionDir = storage.getSessionDir(user, session) // Assert + log.info("Asserting session directory is of type File") assertTrue(sessionDir is File, "Expected File type for session directory") + log.info("testGetSessionDir completed successfully") } @Test fun testGetSessionName() { + log.info("Starting testGetSessionName") // Arrange val user = User(email = "test@example.com") val session = Session("G-20230101-1234") // Act + log.debug("Getting session name for user: {} and session: {}", user.email, session.sessionId) val sessionName = storage.getSessionName(user, session) // Assert + log.info("Asserting session name is not null and is of type String") Assertions.assertNotNull(sessionName) assertTrue(sessionName is String) + log.info("testGetSessionName completed successfully") } @Test fun testGetSessionTime() { + log.info("Starting testGetSessionTime") // Arrange val user = User(email = "test@example.com") val session = Session("G-20230101-1234") + log.debug("Updating message for user: {} and session: {}", user.email, session.sessionId) storage.updateMessage(user, session, "msg001", "

Hello, World!

Hello, World!

") // Act + log.debug("Getting session time for user: {} and session: {}", user.email, session.sessionId) val sessionTime = storage.getSessionTime(user, session) // Assert + log.info("Asserting session time is not null and is of type Date") Assertions.assertNotNull(sessionTime) assertTrue(sessionTime is Date) + log.info("testGetSessionTime completed successfully") } @Test fun testListSessions() { + log.info("Starting testListSessions") // Arrange val user = User(email = "test@example.com") // Act + log.debug("Listing sessions for user: {}", user.email) val sessions = storage.listSessions(user, "") // Assert + log.info("Asserting sessions list is not null and is of type List") Assertions.assertNotNull(sessions) assertTrue(sessions is List<*>) + log.info("testListSessions completed successfully") } @Test fun testSetJson() { + log.info("Starting testSetJson") // Arrange val user = User(email = "test@example.com") val session = Session("G-20230101-1234") @@ -108,15 +137,19 @@ abstract class StorageInterfaceTest(val storage: StorageInterface) { val settings = mapOf("theme" to "dark") // Act + log.debug("Setting JSON for user: {} and session: {}", user.email, session.sessionId) val result = storage.setJson(user, session, filename, settings) // Assert + log.info("Asserting JSON setting result is not null and matches input") Assertions.assertNotNull(result) assertEquals(settings, result) + log.info("testSetJson completed successfully") } @Test fun testUpdateMessage() { + log.info("Starting testUpdateMessage") // Arrange val user = User(email = "test@example.com") val session = Session("G-20230101-1234") @@ -125,53 +158,70 @@ abstract class StorageInterfaceTest(val storage: StorageInterface) { // Act and Assert try { + log.debug("Updating message for user: {} and session: {}", user.email, session.sessionId) storage.updateMessage(user, session, messageId, value) + log.info("Message updated successfully") // If no exception is thrown, the test passes. } catch (e: Exception) { + log.error("Exception thrown while updating message", e) Assertions.fail("Exception should not be thrown") } + log.info("testUpdateMessage completed successfully") } @Test fun testListSessionsWithDir() { + log.info("Starting testListSessionsWithDir") // Arrange val directory = File(System.getProperty("user.dir")) // Example directory // Act + log.debug("Listing sessions for directory: {}", directory.absolutePath) val sessionList = storage.listSessions(directory, "") // Assert + log.info("Asserting session list is not null and is of type List") Assertions.assertNotNull(sessionList) assertTrue(sessionList is List<*>) + log.info("testListSessionsWithDir completed successfully") } @Test fun testUserRoot() { + log.info("Starting testUserRoot") // Arrange val user = User(email = "test@example.com") // Act + log.debug("Getting user root for user: {}", user.email) val userRoot = storage.userRoot(user) // Assert + log.info("Asserting user root is not null and is of type File") Assertions.assertNotNull(userRoot) assertTrue(userRoot is File) + log.info("testUserRoot completed successfully") } @Test fun testDeleteSession() { + log.info("Starting testDeleteSession") // Arrange val user = User(email = "test@example.com") val session = Session("G-20230101-1234") // Act and Assert try { + log.debug("Deleting session for user: {} and session: {}", user.email, session.sessionId) storage.deleteSession(user, session) + log.info("Session deleted successfully") // If no exception is thrown, the test passes. } catch (e: Exception) { + log.error("Exception thrown while deleting session", e) Assertions.fail("Exception should not be thrown") } + log.info("testDeleteSession completed successfully") } // Continue writing tests for each method in StorageInterface... // ... -} +} \ No newline at end of file diff --git a/core/src/main/kotlin/com/simiacryptus/skyenet/core/platform/test/UsageTest.kt b/core/src/main/kotlin/com/simiacryptus/skyenet/core/platform/test/UsageTest.kt index 6b2b6b29..af5b3e29 100644 --- a/core/src/main/kotlin/com/simiacryptus/skyenet/core/platform/test/UsageTest.kt +++ b/core/src/main/kotlin/com/simiacryptus/skyenet/core/platform/test/UsageTest.kt @@ -11,6 +11,10 @@ import org.junit.jupiter.api.Test import kotlin.random.Random abstract class UsageTest(private val impl: UsageInterface) { + companion object { + private val log = org.slf4j.LoggerFactory.getLogger(UsageTest::class.java) + } + private val testUser = User( email = "test@example.com", name = "Test User", @@ -19,11 +23,13 @@ abstract class UsageTest(private val impl: UsageInterface) { @BeforeEach fun setup() { + log.info("Setting up UsageTest: Clearing all usage data") impl.clear() } @Test fun `incrementUsage should increment usage for session`() { + log.debug("Starting test: incrementUsage should increment usage for session") val model = OpenAIModels.GPT4oMini val session = Session.newGlobalID() val usage = ApiModel.Usage( @@ -31,6 +37,7 @@ abstract class UsageTest(private val impl: UsageInterface) { completion_tokens = 20, cost = 30.0, ) + log.info("Incrementing usage for session {} with model {}", session, model) impl.incrementUsage(session, testUser, model, usage) val usageSummary = impl.getSessionUsageSummary(session) Assertions.assertEquals(usage, usageSummary[model]) @@ -40,6 +47,7 @@ abstract class UsageTest(private val impl: UsageInterface) { @Test fun `getUserUsageSummary should return correct usage summary`() { + log.debug("Starting test: getUserUsageSummary should return correct usage summary") val model = OpenAIModels.GPT4oMini val session = Session.newGlobalID() val usage = ApiModel.Usage( @@ -47,6 +55,7 @@ abstract class UsageTest(private val impl: UsageInterface) { completion_tokens = 25, cost = 35.0, ) + log.info("Incrementing usage for user {} with model {}", testUser.email, model) impl.incrementUsage(session, testUser, model, usage) val userUsageSummary = impl.getUserUsageSummary(testUser) Assertions.assertEquals(usage, userUsageSummary[model]) @@ -54,6 +63,7 @@ abstract class UsageTest(private val impl: UsageInterface) { @Test fun `clear should reset all usage data`() { + log.debug("Starting test: clear should reset all usage data") val model = OpenAIModels.GPT4oMini val session = Session.newGlobalID() val usage = ApiModel.Usage( @@ -61,7 +71,9 @@ abstract class UsageTest(private val impl: UsageInterface) { completion_tokens = 30, cost = 40.0, ) + log.info("Incrementing usage before clearing") impl.incrementUsage(session, testUser, model, usage) + log.info("Clearing all usage data") impl.clear() val usageSummary = impl.getSessionUsageSummary(session) Assertions.assertTrue(usageSummary.isEmpty()) @@ -71,6 +83,7 @@ abstract class UsageTest(private val impl: UsageInterface) { @Test fun `incrementUsage should handle multiple models correctly`() { + log.debug("Starting test: incrementUsage should handle multiple models correctly") val model1 = OpenAIModels.GPT4oMini val model2 = OpenAIModels.GPT4Turbo val session = Session.newGlobalID() @@ -84,8 +97,10 @@ abstract class UsageTest(private val impl: UsageInterface) { completion_tokens = 10, cost = 15.0, ) + log.info("Incrementing usage for model1 {} and model2 {}", model1, model2) impl.incrementUsage(session, testUser, model1, usage1) impl.incrementUsage(session, testUser, model2, usage2) + log.debug("Verifying usage summaries for session and user") val usageSummary = impl.getSessionUsageSummary(session) Assertions.assertEquals(usage1, usageSummary[model1]) Assertions.assertEquals(usage2, usageSummary[model2]) @@ -96,6 +111,7 @@ abstract class UsageTest(private val impl: UsageInterface) { @Test fun `incrementUsage should accumulate usage for the same model`() { + log.debug("Starting test: incrementUsage should accumulate usage for the same model") val model = OpenAIModels.GPT4oMini val session = Session.newGlobalID() val usage1 = ApiModel.Usage( @@ -108,8 +124,10 @@ abstract class UsageTest(private val impl: UsageInterface) { completion_tokens = 10, cost = 15.0, ) + log.info("Incrementing usage twice for model {}", model) impl.incrementUsage(session, testUser, model, usage1) impl.incrementUsage(session, testUser, model, usage2) + log.debug("Verifying accumulated usage") val usageSummary = impl.getSessionUsageSummary(session) val expectedUsage = ApiModel.Usage( prompt_tokens = 15, @@ -123,18 +141,22 @@ abstract class UsageTest(private val impl: UsageInterface) { @Test fun `getSessionUsageSummary should return empty map for unknown session`() { + log.debug("Starting test: getSessionUsageSummary should return empty map for unknown session") val session = Session.newGlobalID() + log.info("Retrieving usage summary for unknown session {}", session) val usageSummary = impl.getSessionUsageSummary(session) Assertions.assertTrue(usageSummary.isEmpty()) } @Test fun `getUserUsageSummary should return empty map for unknown user`() { + log.debug("Starting test: getUserUsageSummary should return empty map for unknown user") val unknownUser = User( email = "unknown@example.com", name = "Unknown User", id = Random.nextInt().toString() ) + log.info("Retrieving usage summary for unknown user {}", unknownUser.email) val userUsageSummary = impl.getUserUsageSummary(unknownUser) Assertions.assertTrue(userUsageSummary.isEmpty()) } diff --git a/core/src/main/kotlin/com/simiacryptus/skyenet/core/platform/test/UserSettingsTest.kt b/core/src/main/kotlin/com/simiacryptus/skyenet/core/platform/test/UserSettingsTest.kt index 330f7b74..9227c22d 100644 --- a/core/src/main/kotlin/com/simiacryptus/skyenet/core/platform/test/UserSettingsTest.kt +++ b/core/src/main/kotlin/com/simiacryptus/skyenet/core/platform/test/UserSettingsTest.kt @@ -8,38 +8,56 @@ import org.junit.jupiter.api.Test import java.util.* abstract class UserSettingsTest(private val userSettings: UserSettingsInterface) { + companion object { + private val log = org.slf4j.LoggerFactory.getLogger(UserSettingsTest::class.java) + } @Test fun `updateUserSettings should store custom settings for user`() { + log.info("Starting test: updateUserSettings should store custom settings for user") val id = UUID.randomUUID().toString() val testUser = User( email = "$id@example.com", name = "Test User", id = id ) + log.debug("Created test user with id: {}", id) val newSettings = UserSettingsInterface.UserSettings(apiKeys = mapOf(APIProvider.OpenAI to "12345")) + log.debug("Updating user settings with new API key") userSettings.updateUserSettings(testUser, newSettings) + val settings = userSettings.getUserSettings(testUser) + log.debug("Retrieved user settings after update") + Assertions.assertEquals("12345", settings.apiKeys[APIProvider.OpenAI]) + log.info("Test completed: updateUserSettings successfully stored custom settings for user") } @Test fun `getUserSettings should return updated settings after updateUserSettings is called`() { + log.info("Starting test: getUserSettings should return updated settings after updateUserSettings is called") val id = UUID.randomUUID().toString() val testUser = User( email = "$id@example.com", name = "Test User", id = id ) + log.debug("Created test user with id: {}", id) + val initialSettings = userSettings.getUserSettings(testUser) + log.debug("Retrieved initial user settings") Assertions.assertEquals("", initialSettings.apiKeys[APIProvider.OpenAI]) val updatedSettings = UserSettingsInterface.UserSettings(apiKeys = mapOf(APIProvider.OpenAI to "67890")) + log.debug("Updating user settings with new API key") userSettings.updateUserSettings(testUser, updatedSettings) val settingsAfterUpdate = userSettings.getUserSettings(testUser) + log.debug("Retrieved user settings after update") + Assertions.assertEquals("67890", settingsAfterUpdate.apiKeys[APIProvider.OpenAI]) + log.info("Test completed: getUserSettings successfully returned updated settings after updateUserSettings was called") } } \ No newline at end of file diff --git a/gradle.properties b/gradle.properties index 691db35a..c690983e 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,5 +1,5 @@ # Gradle Releases -> https://github.com/gradle/gradle/releases libraryGroup=com.simiacryptus.skyenet -libraryVersion=1.2.13 +libraryVersion=1.2.14 gradleVersion=7.6.1 kotlin.daemon.jvmargs=-Xmx4g diff --git a/webui/build.gradle.kts b/webui/build.gradle.kts index 70e01c6d..c6dc03ce 100644 --- a/webui/build.gradle.kts +++ b/webui/build.gradle.kts @@ -36,7 +36,7 @@ val jackson_version = "2.17.2" dependencies { - implementation(group = "com.simiacryptus", name = "jo-penai", version = "1.1.10") { + implementation(group = "com.simiacryptus", name = "jo-penai", version = "1.1.11") { exclude(group = "org.slf4j") } @@ -45,7 +45,7 @@ dependencies { implementation("org.apache.pdfbox:pdfbox:2.0.27") compileOnly("org.seleniumhq.selenium:selenium-chrome-driver:4.16.1") - compileOnly("org.jsoup:jsoup:1.17.2") + implementation("org.jsoup:jsoup:1.18.1") implementation("com.google.zxing:core:3.5.3") implementation("com.google.zxing:javase:3.5.3") @@ -89,6 +89,8 @@ dependencies { implementation(group = "com.fasterxml.jackson.core", name = "jackson-annotations", version = jackson_version) implementation(group = "com.fasterxml.jackson.module", name = "jackson-module-kotlin", version = jackson_version) + +// implementation(group = "com.google.apis", name = "google-api-services-customsearch", version = "v1-rev20230702-2.0.0") compileOnly(group = "com.google.api-client", name = "google-api-client", version = "1.35.2" /*"1.35.2"*/) compileOnly(group = "com.google.oauth-client", name = "google-oauth-client-jetty", version = "1.34.1") compileOnly(group = "com.google.apis", name = "google-api-services-oauth2", version = "v2-rev157-1.25.0") diff --git a/webui/src/main/kotlin/com/simiacryptus/diff/AddApplyFileDiffLinks.kt b/webui/src/main/kotlin/com/simiacryptus/diff/AddApplyFileDiffLinks.kt index f0807caa..60bf87e6 100644 --- a/webui/src/main/kotlin/com/simiacryptus/diff/AddApplyFileDiffLinks.kt +++ b/webui/src/main/kotlin/com/simiacryptus/diff/AddApplyFileDiffLinks.kt @@ -3,15 +3,19 @@ package com.simiacryptus.diff import com.simiacryptus.diff.FileValidationUtils.Companion.isGitignore import com.simiacryptus.jopenai.API import com.simiacryptus.jopenai.OpenAIClient +import com.simiacryptus.jopenai.models.ChatModel import com.simiacryptus.jopenai.models.OpenAIModels import com.simiacryptus.skyenet.AgentPatterns.displayMapInTabs +import com.simiacryptus.skyenet.core.actors.ParsedActor import com.simiacryptus.skyenet.core.actors.SimpleActor import com.simiacryptus.skyenet.set import com.simiacryptus.skyenet.util.MarkdownUtil.renderMarkdown import com.simiacryptus.skyenet.webui.application.ApplicationInterface import com.simiacryptus.skyenet.webui.session.SocketManagerBase +import com.simiacryptus.util.JsonUtil import java.io.File import java.nio.file.Path +import java.util.concurrent.atomic.AtomicReference import kotlin.io.path.readText @@ -26,6 +30,7 @@ fun SocketManagerBase.addApplyFileDiffLinks( ui: ApplicationInterface, api: API, shouldAutoApply: (Path) -> Boolean = { false }, + model: ChatModel? = null, ): String { // Check if there's an unclosed code block and close it if necessary val initiator = "(?s)```\\w*\n".toRegex() @@ -36,10 +41,10 @@ fun SocketManagerBase.addApplyFileDiffLinks( response + "\n```\n", handle, ui, - api + api, + model = model, ) } - // Define regex patterns for headers and code blocks val headerPattern = """(? + PatchOrCode( + id = "patch_" + index.toString(), + type = "patch", + data = it.second.groupValues[2] + ) + } + codeblocks.mapIndexed { index, it -> + PatchOrCode( + id = "code_" + index.toString(), + type = "code", + data = it.second.groupValues[2] + ) + } + val corrections = if(model == null) null else try { + ParsedActor( + resultClass = CorrectedPatchAndCodeList::class.java, + exampleInstance = CorrectedPatchAndCodeList(listOf( + CorrectedPatchOrCode("patch_0", "src/utils/exampleUtils.js"), + CorrectedPatchOrCode("code_0", "src/utils/exampleUtils.js"), + CorrectedPatchOrCode("patch_1", "tests/exampleUtils.test.js"), + CorrectedPatchOrCode("code_1", "tests/exampleUtils.test.js"), + )), + prompt = """ + Review and correct the file path assignments for the following patches and code blocks. + """.trimIndent(), + model = model, + temperature = 0.0, + parsingModel = model, + ).getParser(api).apply(listOf( + response, + JsonUtil.toJson( + PatchAndCodeList( + changes = changes + ) + ) + ).joinToString("\n\n")).changes?.associateBy { it.id }?.mapValues { it.value.filename } ?: emptyMap() + } catch (e: Throwable) { + log.error("Error consulting AI for corrections", e) + null + } + // Process diff blocks and add patch links - val withPatchLinks: String = patchBlocks.fold(response) { markdown, diffBlock -> + val withPatchLinks: String = patchBlocks.foldIndexed(response) { index, markdown, diffBlock -> val value = diffBlock.second.groupValues[2].trim() - val header = headers.lastOrNull { it.first.last < diffBlock.first.first } - val filename = resolve(root, header?.second ?: "Unknown") + var header = headers.lastOrNull { it.first.last < diffBlock.first.first }?.second ?: "Unknown" + header = corrections?.get("patch_$index") ?: header + val filename = resolve(root, header) val newValue = renderDiffBlock(root, filename, value, handle, ui, api, shouldAutoApply) markdown.replace(diffBlock.second.value, newValue) } // Process code blocks and add save links - val withSaveLinks = codeblocks.fold(withPatchLinks) { markdown, codeBlock -> + val withSaveLinks = codeblocks.foldIndexed(withPatchLinks) { index, markdown, codeBlock -> val lang = codeBlock.second.groupValues[1] val value = codeBlock.second.groupValues[2].trim() - val header = headers.lastOrNull { it.first.last < codeBlock.first.first }?.second + var header = headers.lastOrNull { it.first.last < codeBlock.first.first }?.second + header = corrections?.get("code_$index") ?: header val newMarkdown = renderNewFile(header, root, ui, shouldAutoApply, value, handle, lang) markdown.replace(codeBlock.second.value, newMarkdown) } return withSaveLinks } +data class PatchAndCodeList( + val changes: List, +) + +data class PatchOrCode( + val id: String? = null, + val type: String? = null, + val filename: String? = null, + val data: String? = null, +) + +data class CorrectedPatchAndCodeList( + val changes: List? = null, +) + +data class CorrectedPatchOrCode( + val id: String? = null, + val filename: String? = null, +) + private fun SocketManagerBase.renderNewFile( header: String?, root: Path, @@ -206,6 +276,7 @@ private fun SocketManagerBase.renderDiffBlock( ui: ApplicationInterface, api: API?, shouldAutoApply: (Path) -> Boolean, + model: ChatModel? = null, ): String { val filepath = root.resolve(filename) @@ -225,17 +296,43 @@ private fun SocketManagerBase.renderDiffBlock( renderMarkdown("```\n${e.stackTraceToString()}\n```\n", ui = ui) } + // Function to create a revert button + fun createRevertButton(filepath: Path, originalCode: String, handle: (Map) -> Unit): String { + val relativize = try { + root.relativize(filepath) + } catch (e: Throwable) { + filepath + } + val revertTask = ui.newTask(false) + lateinit var revertButton: StringBuilder + revertButton = revertTask.complete(hrefLink("Revert", classname = "href-link cmd-button") { + try { + filepath.toFile().writeText(originalCode, Charsets.UTF_8) + handle(mapOf(relativize to originalCode)) + revertButton.set("""
Reverted
""") + revertTask.complete() + } catch (e: Throwable) { + revertButton.append("""
Error: ${e.message}
""") + revertTask.error(null, e) + } + })!! + return revertTask.placeholder + } + if (echoDiff.isNotBlank() && newCode.isValid && shouldAutoApply(filepath ?: root.resolve(filename))) { try { filepath.toFile().writeText(newCode.newCode, Charsets.UTF_8) + val originalCode = AtomicReference(prevCode) handle(mapOf(relativize to newCode.newCode)) - return "```diff\n$diffVal\n```\n" + """
Diff Automatically Applied to ${filepath}
""" + val revertButton = createRevertButton(filepath, originalCode.get(), handle) + return "```diff\n$diffVal\n```\n" + """
Diff Automatically Applied to ${filepath}
""" + revertButton } catch (e: Throwable) { log.error("Error auto-applying diff", e) return "```diff\n$diffVal\n```\n" + """
Error Auto-Applying Diff to ${filepath}: ${e.message}
""" } } + val diffTask = ui.newTask(root = false) diffTask.complete(renderMarkdown("```diff\n$diffVal\n```\n", ui = ui)) @@ -373,7 +470,7 @@ Please provide a fix for the diff above in the form of a diff patch. """ ), api as OpenAIClient ) - answer = ui.socketManager?.addApplyFileDiffLinks(root, answer, handle, ui, api) ?: answer + answer = ui.socketManager?.addApplyFileDiffLinks(root, answer, handle, ui, api, model=model) ?: answer header?.clear() fixTask.complete(renderMarkdown(answer)) } catch (e: Throwable) { diff --git a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/general/AutoPlanChatApp.kt b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/general/AutoPlanChatApp.kt index 2b14ee67..7d2af73f 100644 --- a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/general/AutoPlanChatApp.kt +++ b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/general/AutoPlanChatApp.kt @@ -2,7 +2,7 @@ package com.simiacryptus.skyenet.apps.general import com.simiacryptus.jopenai.API import com.simiacryptus.jopenai.ChatClient -import com.simiacryptus.jopenai.models.TextModel +import com.simiacryptus.jopenai.models.ChatModel import com.simiacryptus.skyenet.TabbedDisplay import com.simiacryptus.skyenet.apps.plan.PlanCoordinator import com.simiacryptus.skyenet.apps.plan.PlanSettings @@ -12,8 +12,10 @@ import com.simiacryptus.skyenet.apps.plan.file.FileModificationTask.FileModifica import com.simiacryptus.skyenet.core.actors.ParsedActor import com.simiacryptus.skyenet.core.platform.Session import com.simiacryptus.skyenet.core.platform.model.User +import com.simiacryptus.skyenet.set import com.simiacryptus.skyenet.util.MarkdownUtil.renderMarkdown import com.simiacryptus.skyenet.webui.application.ApplicationInterface +import com.simiacryptus.skyenet.webui.session.SessionTask import com.simiacryptus.util.JsonUtil import org.slf4j.LoggerFactory import java.io.File @@ -26,13 +28,14 @@ open class AutoPlanChatApp( applicationName: String = "Auto Plan Chat App", path: String = "/autoPlanChat", planSettings: PlanSettings, - model: TextModel, - parsingModel: TextModel, + model: ChatModel, + parsingModel: ChatModel, domainName: String = "localhost", showMenubar: Boolean = true, api: API? = null, val maxTaskHistoryChars: Int = 20000, - val maxTasksPerIteration: Int = 3 + val maxTasksPerIteration: Int = 3, + val maxIterations: Int = 100 ) : PlanChatApp( applicationName = applicationName, path = path, @@ -46,7 +49,7 @@ open class AutoPlanChatApp( override val stickyInput = true override val singleInput = false companion object { - private val log = LoggerFactory.getLogger(AutoPlanChatApp::class.java) + private val log = org.slf4j.LoggerFactory.getLogger(AutoPlanChatApp::class.java) } data class ThinkingStatus( @@ -78,26 +81,31 @@ open class AutoPlanChatApp( val description: String? = null ) - private val thinkingStatus = AtomicReference(null) - private lateinit var initActor: ParsedActor - private lateinit var updateActor: ParsedActor - private val currentUserMessage = AtomicReference(null) - data class ExecutionRecord( val time: Date? = Date(), val task: PlanTaskBase? = null, - val result: Any? = null + val result: String? = null ) data class Tasks( val tasks: MutableList? = null ) + private val currentUserMessage = AtomicReference(null) + private var isRunning = false val executionRecords = mutableListOf() - override fun userMessage(session: Session, user: User?, userMessage: String, ui: ApplicationInterface, api: API) { + + override fun userMessage( + session: Session, + user: User?, + userMessage: String, + ui: ApplicationInterface, + api: API + ) { try { log.info("Received user message: $userMessage") - if (thinkingStatus.get() == null) { + if (!isRunning) { + isRunning = true log.info("Starting new auto plan chat") startAutoPlanChat(session, user, userMessage, ui, api) } else { @@ -112,7 +120,14 @@ open class AutoPlanChatApp( } } - private fun startAutoPlanChat(session: Session, user: User?, userMessage: String, ui: ApplicationInterface, api: API) { + protected open fun startAutoPlanChat( + session: Session, + user: User?, + userMessage: String, + ui: ApplicationInterface, + api: API + ) { + val thinkingStatus = AtomicReference(null); val task = ui.newTask(true) val api = (api as ChatClient).getChildClient().apply { val createFile = task.createFile(".logs/api-${UUID.randomUUID()}.log") @@ -124,8 +139,19 @@ open class AutoPlanChatApp( val tabbedDisplay = TabbedDisplay(task) val executor = ui.socketManager!!.pool + var continueLoop = true executor.execute { try { + ui.newTask(false).let { task -> + tabbedDisplay["Controls"] = task.placeholder + lateinit var stopLink: StringBuilder + stopLink = task.add(ui.hrefLink("Stop") { + continueLoop = false + executor.shutdown() + stopLink.set("Stopped") + task.complete() + })!! + } tabbedDisplay.update() task.complete() @@ -142,78 +168,13 @@ open class AutoPlanChatApp( root = planSettings.workingDir?.let { File(it).toPath() } ?: dataStorage.getDataDir(user, session).toPath(), planSettings = planSettings ) - initActor = ParsedActor( - name = "ThinkingStatusInitializer", - resultClass = ThinkingStatus::class.java, - exampleInstance = ThinkingStatus( - initialPrompt = "Example prompt", - goals = Goals( - shortTerm = mutableListOf("Understand the user's request"), - longTerm = mutableListOf("Complete the user's task") - ), - knowledge = Knowledge( - facts = mutableListOf("Initial Context: User's request received"), - openQuestions = mutableListOf("What is the first task?") - ), - executionContext = ExecutionContext( - nextSteps = mutableListOf("Analyze the initial prompt", "Identify key objectives"), - ) - ), - prompt = """ - Given the user's initial prompt, initialize the thinking status for an AI assistant. - Set short-term and long-term goals. - Generate relevant open questions and hypotheses to guide the planning process. - Initialize the knowledge base with any relevant information from the initial prompt. - Set up the execution context with initial next steps, progress (0-100), estimated time remaining, and confidence level (0-100). - Identify potential challenges and available resources. - """.trimIndent(), - model = planSettings.defaultModel, - parsingModel = planSettings.parsingModel, - temperature = planSettings.temperature, - describer = planSettings.describer() - ) - updateActor = ParsedActor( - name = "UpdateQuestionsActor", - resultClass = ThinkingStatus::class.java, - exampleInstance = ThinkingStatus( - initialPrompt = "Example prompt", - goals = Goals( - shortTerm = mutableListOf("Analyze task results"), - longTerm = mutableListOf("Complete the user's request") - ), - knowledge = Knowledge( - facts = mutableListOf( - "Initial Context: User's request received", - "Task 1 Result: Analyzed user's request" - ), - openQuestions = mutableListOf("What is the next task?", "Are there any remaining tasks?") - ), - executionContext = ExecutionContext( - completedTasks = mutableListOf("task_1"), - nextSteps = mutableListOf("Analyze task results", "Determine next action"), - ) - ), - prompt = """ - Given the current thinking status, the last completed task, and its result, - update the open questions to guide the next steps of the planning process. - Consider what information is still needed and what new questions arise from the task result. - Update the current goal if necessary, adjust the progress, suggest next steps, and add any new insights. - Update the knowledge base with new facts and hypotheses. - Update the estimated time remaining and adjust the confidence level based on progress. - Reassess challenges, available resources, and alternative approaches. - """.trimIndent(), - model = planSettings.defaultModel, - parsingModel = planSettings.parsingModel, - temperature = planSettings.temperature, - describer = planSettings.describer() - ) - val initialStatus = initActor.answer(initialPrompt(userMessage), this.api!!).obj + val initialStatus = initThinking(planSettings, userMessage) initialStatus.initialPrompt = userMessage thinkingStatus.set(initialStatus) initialPromptTask.complete(renderMarkdown("Initial Thinking Status:\n${formatThinkingStatus(thinkingStatus.get()!!)}")) var iteration = 0 - while (iteration++ < 100) { + while (iteration++ < maxIterations && continueLoop) { task.complete() val task = ui.newTask(false).apply { tabbedDisplay["Iteration $iteration"] = placeholder } val api = api.getChildClient().apply { @@ -224,65 +185,17 @@ open class AutoPlanChatApp( } } val tabbedDisplay = TabbedDisplay(task) - tabbedDisplay.update() + ui.newTask(false).apply { + tabbedDisplay["Inputs"] = placeholder + header("Project Info") + contextData().forEach { add(renderMarkdown(it)) } + header("Evaluation Records") + formatEvalRecords().forEach { add(renderMarkdown(it)) } + header("Current Thinking Status") + formatThinkingStatus(thinkingStatus.get()!!).let { add(renderMarkdown(it)) } + } val nextTask = try { - val describer1 = planSettings.describer() - val chooserResult = ParsedActor( - name = "SingleTaskChooser", - resultClass = Tasks::class.java, - exampleInstance = Tasks( - listOf( - FileModificationTaskData( - task_description = "Modify the file 'example.txt' to include the given input." - ) - ).toMutableList() - ), - prompt = """ - Given the following input, choose up to $maxTasksPerIteration tasks to execute. Do not create a full plan, just select the most appropriate task types for the given input. - Available task types: - - ${ - TaskType.Companion.getAvailableTaskTypes(coordinator.planSettings).joinToString>("\n") { taskType -> - "* ${TaskType.Companion.getImpl(coordinator.planSettings, taskType).promptSegment()}" - } - } - - Choose the most suitable task types and provide details of how they should be executed. - """.trimIndent(), - model = coordinator.planSettings.defaultModel, - parsingModel = coordinator.planSettings.parsingModel, - temperature = coordinator.planSettings.temperature, - describer = describer1, - parserPrompt = """ - Task Subtype Schema: - - ${ - TaskType.Companion.getAvailableTaskTypes(coordinator.planSettings).joinToString>("\n\n") { taskType -> - """ - ${taskType.name}: - ${describer1.describe(taskType.taskDataClass).replace("\n", "\n ")} - """.trim() - } - } - """.trimIndent() - ).answer( - initialPrompt(userMessage) + - listOf( - """ - Current thinking status: ${formatThinkingStatus(thinkingStatus.get()!!)} - Please choose the next single task to execute based on the current status. - If there are no tasks to execute, return {}. - """.trimIndent() - ) - + formatEvalRecords(), api - ).obj - chooserResult.tasks?.take(maxTasksPerIteration)?.mapNotNull { task -> - (if (task.task_type == null) { - null - } else { - TaskType.Companion.getImpl(coordinator.planSettings, task) - })?.planTask - } + getNextTask(api, planSettings, coordinator, userMessage, thinkingStatus.get()) } catch (e: Exception) { log.error("Error choosing next task", e) tabbedDisplay["Errors"]?.append(renderMarkdown("Error choosing next task: ${e.message}")) @@ -295,7 +208,7 @@ open class AutoPlanChatApp( val taskResults = mutableListOf>>() for ((index, currentTask) in nextTask.withIndex()) { - val currentTaskId = "task_${(thinkingStatus.get()!!.executionContext?.completedTasks?.size ?: 0) + index + 1}" + val currentTaskId = "task_${index + 1}" val taskExecutionTask = ui.newTask(false) taskExecutionTask.add( renderMarkdown( @@ -307,27 +220,7 @@ open class AutoPlanChatApp( tabbedDisplay["Task Execution $currentTaskId"] = taskExecutionTask.placeholder val future = executor.submit { try { - val api = api.getChildClient().apply { - val createFile = task.createFile(".logs/api-${UUID.randomUUID()}.log") - createFile.second?.apply { - logStreams += this.outputStream().buffered() - task.verbose("API log: $this") - } - } - val taskImpl = TaskType.Companion.getImpl(coordinator.planSettings, currentTask) - val result = StringBuilder() - taskImpl.run( - agent = coordinator, - taskId = currentTaskId, - messages = listOf( - userMessage, - "Current thinking status:\n${formatThinkingStatus(thinkingStatus.get()!!)}" - ) + formatEvalRecords(), - task = taskExecutionTask, - api = api, - resultFn = { result.append(it) } - ) - result.toString() + runTask(api, task, coordinator, currentTask, currentTaskId, userMessage, taskExecutionTask, thinkingStatus.get()) } catch (e: Exception) { taskExecutionTask.error(ui, e) log.error("Error executing task", e) @@ -347,27 +240,13 @@ open class AutoPlanChatApp( executionRecords.addAll(completedTasks) val thinkingStatusTask = ui.newTask(false).apply { tabbedDisplay["Thinking Status"] = placeholder } - //thinkingStatusTask.add(renderMarkdown("Updating Thinking Status:\n${formatThinkingStatus(thinkingStatus.get()!!)}")) - this.thinkingStatus.set(updateActor.answer( - initialPrompt("Current thinking status: ${formatThinkingStatus(this.thinkingStatus.get()!!)}") + - completedTasks.flatMap { record -> - listOf( - "Completed task: ${record.task?.task_description}", - "Task result: ${record.result}" - ) - } + (currentUserMessage.get()?.let> { listOf("User message: $it") } ?: listOf()), - api - ).obj.apply { - this@AutoPlanChatApp.currentUserMessage.set(null) - knowledge?.facts?.apply { - this.addAll(completedTasks.mapIndexed { index, (task, result) -> - "Task ${(executionContext?.completedTasks?.size ?: 0) + index + 1} Result: $result" - }) - } - }) + thinkingStatus.set( + updateThinking(api, planSettings, thinkingStatus.get(), completedTasks) + ) thinkingStatusTask.complete(renderMarkdown("Updated Thinking Status:\n${formatThinkingStatus(thinkingStatus.get()!!)}")) } - } catch (e: Exception) { + task.complete("Auto Plan Chat completed.") + } catch (e: Throwable) { task.error(ui, e) log.error("Error in startAutoPlanChat", e) } finally { @@ -384,11 +263,230 @@ open class AutoPlanChatApp( } - private fun formatEvalRecords(maxTotalLength: Int = maxTaskHistoryChars): List { + protected open fun runTask( + api: ChatClient, + task: SessionTask, + coordinator: PlanCoordinator, + currentTask: PlanTaskBase, + currentTaskId: String, + userMessage: String, + taskExecutionTask: SessionTask, + thinkingStatus: ThinkingStatus? + ): String { + val api = api.getChildClient().apply { + val createFile = task.createFile(".logs/api-${UUID.randomUUID()}.log") + createFile.second?.apply { + logStreams += this.outputStream().buffered() + task.verbose("API log: $this") + } + } + val taskImpl = TaskType.Companion.getImpl(coordinator.planSettings, currentTask) + val result = StringBuilder() + taskImpl.run( + agent = coordinator, + messages = listOf( + userMessage, + "Current thinking status:\n${formatThinkingStatus(thinkingStatus!!)}" + ) + formatEvalRecords(), + task = taskExecutionTask, + api = api, + resultFn = { result.append(it) } + ) + return result.toString() + } + + protected open fun getNextTask( + api: ChatClient, + planSettings: PlanSettings, + coordinator: PlanCoordinator, + userMessage: String, + thinkingStatus: ThinkingStatus? + ): List? { + val describer1 = planSettings.describer() + val tasks = ParsedActor( + name = "SingleTaskChooser", + resultClass = Tasks::class.java, + exampleInstance = Tasks( + listOf( + FileModificationTaskData( + task_description = "Modify the file 'example.txt' to include the given input." + ) + ).toMutableList() + ), + prompt = """ + Given the following input, choose up to ${maxTasksPerIteration} tasks to execute. Do not create a full plan, just select the most appropriate task types for the given input. + Available task types: + + ${ + TaskType.Companion.getAvailableTaskTypes(coordinator.planSettings).joinToString>("\n") { taskType -> + "* ${TaskType.Companion.getImpl(coordinator.planSettings, taskType).promptSegment()}" + } + } + + Choose the most suitable task types and provide details of how they should be executed. + """.trimIndent(), + model = coordinator.planSettings.defaultModel, + parsingModel = coordinator.planSettings.parsingModel, + temperature = coordinator.planSettings.temperature, + describer = describer1, + parserPrompt = """ + Task Subtype Schema: + + ${ + TaskType.Companion.getAvailableTaskTypes(coordinator.planSettings).joinToString>("\n\n") { taskType -> + """ + ${taskType.name}: + ${describer1.describe(taskType.taskDataClass).replace("\n", "\n ")} + """.trim() + } + } + """.trimIndent() + ).answer( + listOf(userMessage) + contextData() + + listOf( + """ + Current thinking status: ${formatThinkingStatus(thinkingStatus!!)} + Please choose the next single task to execute based on the current status. + If there are no tasks to execute, return {}. + """.trimIndent() + ) + + formatEvalRecords(), api + ).obj.tasks?.map { task -> + task to (if (task.task_type == null) { + null + } else { + TaskType.Companion.getImpl(coordinator.planSettings, task) + })?.planTask + } + if (tasks.isNullOrEmpty()) { + log.info("No tasks selected from: ${tasks?.map { it.first }}") + return null + } else if (tasks.mapNotNull { it.second }.isEmpty()) { + log.warn("No tasks selected from: ${tasks.map { it.first }}") + return null + } else { + return tasks.mapNotNull { it.second }.take(maxTasksPerIteration) + } + } + + protected open fun updateThinking( + api: ChatClient, + planSettings: PlanSettings, + thinkingStatus: ThinkingStatus?, + completedTasks: List + ): ThinkingStatus = ParsedActor( + name = "UpdateQuestionsActor", + resultClass = ThinkingStatus::class.java, + exampleInstance = ThinkingStatus( + initialPrompt = "Example prompt", + goals = Goals( + shortTerm = mutableListOf("Analyze task results"), + longTerm = mutableListOf("Complete the user's request") + ), + knowledge = Knowledge( + facts = mutableListOf( + "Initial Context: User's request received", + "Task 1 Result: Analyzed user's request" + ), + openQuestions = mutableListOf("What is the next task?", "Are there any remaining tasks?") + ), + executionContext = ExecutionContext( + completedTasks = mutableListOf("task_1"), + nextSteps = mutableListOf("Analyze task results", "Determine next action"), + ) + ), + prompt = """ + Given the current thinking status, the last completed task, and its result, + update the open questions to guide the next steps of the planning process. + Consider what information is still needed and what new questions arise from the task result. + Update the current goal if necessary, adjust the progress, suggest next steps, and add any new insights. + Update the knowledge base with new facts and hypotheses. + Update the estimated time remaining and adjust the confidence level based on progress. + Reassess challenges, available resources, and alternative approaches. + """.trimIndent(), + model = planSettings.defaultModel, + parsingModel = planSettings.parsingModel, + temperature = planSettings.temperature, + describer = planSettings.describer() + ).answer( + listOf("Current thinking status: ${formatThinkingStatus(thinkingStatus!!)}") + contextData() + + completedTasks.flatMap { record -> + listOf( + "Completed task: ${record.task?.task_description}", + "Task result: ${record.result}" + ) + } + (currentUserMessage.get()?.let> { listOf("User message: $it") } ?: listOf()), + api + ).obj.apply { + this@AutoPlanChatApp.currentUserMessage.set(null) + knowledge?.facts?.apply { + this.addAll(completedTasks.mapIndexed { index, (task, result) -> + "Task ${(executionContext?.completedTasks?.size ?: 0) + index + 1} Result: $result" + }) + } + } + + protected open fun initThinking( + planSettings: PlanSettings, + userMessage: String + ): ThinkingStatus { + val initialStatus = ParsedActor( + name = "ThinkingStatusInitializer", + resultClass = ThinkingStatus::class.java, + exampleInstance = ThinkingStatus( + initialPrompt = "Example prompt", + goals = Goals( + shortTerm = mutableListOf("Understand the user's request"), + longTerm = mutableListOf("Complete the user's task") + ), + knowledge = Knowledge( + facts = mutableListOf("Initial Context: User's request received"), + openQuestions = mutableListOf("What is the first task?") + ), + executionContext = ExecutionContext( + nextSteps = mutableListOf("Analyze the initial prompt", "Identify key objectives"), + ) + ), + prompt = """ + Given the user's initial prompt, initialize the thinking status for an AI assistant. + Set short-term and long-term goals. + Generate relevant open questions and hypotheses to guide the planning process. + Initialize the knowledge base with any relevant information from the initial prompt. + Set up the execution context with initial next steps, progress (0-100), estimated time remaining, and confidence level (0-100). + Identify potential challenges and available resources. + """.trimIndent(), + model = planSettings.defaultModel, + parsingModel = planSettings.parsingModel, + temperature = planSettings.temperature, + describer = planSettings.describer() + ).answer(listOf(userMessage) + contextData(), this.api!!).obj + return initialStatus + } + + protected open fun formatEvalRecords(maxTotalLength: Int = maxTaskHistoryChars): List { var currentLength = 0 val formattedRecords = mutableListOf() for (record in executionRecords.reversed()) { - val formattedRecord = "Task ${executionRecords.indexOf(record) + 1} Result: ${record.result}" + val formattedRecord = """ +# Task ${executionRecords.indexOf(record) + 1} + +## Task: +```json +${JsonUtil.toJson(record.task!!)} +``` + +## Result: +${record.result?.let { + // Add 2 levels of header level to each header + it.split("\n").joinToString("\n") { line -> + if (line.startsWith("#")) { + "##$line" + } else { + line + } + } +}} +""" if (currentLength + formattedRecord.length > maxTotalLength) { formattedRecords.add("... (earlier records truncated)") break @@ -399,13 +497,12 @@ open class AutoPlanChatApp( return formattedRecords } - private fun formatThinkingStatus(thinkingStatus: ThinkingStatus) = """ + protected open fun formatThinkingStatus(thinkingStatus: ThinkingStatus) = """ ```json ${JsonUtil.toJson(thinkingStatus)} ``` """ - protected open fun initialPrompt(userMessage: String): List = listOf(userMessage) + contextData() protected open fun contextData(): List = emptyList() } \ No newline at end of file diff --git a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/general/CmdPatchApp.kt b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/general/CmdPatchApp.kt index bb67bf65..a319ea74 100644 --- a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/general/CmdPatchApp.kt +++ b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/general/CmdPatchApp.kt @@ -3,6 +3,7 @@ package com.simiacryptus.skyenet.apps.general import com.simiacryptus.diff.FileValidationUtils import com.simiacryptus.jopenai.ChatClient +import com.simiacryptus.jopenai.models.ChatModel import com.simiacryptus.jopenai.models.TextModel import com.simiacryptus.skyenet.core.platform.Session import com.simiacryptus.skyenet.set @@ -18,7 +19,7 @@ class CmdPatchApp( settings: Settings, api: ChatClient, val files: Array?, - model: TextModel + model: ChatModel ) : PatchApp(root.toFile(), session, settings, api, model) { companion object { private val log = LoggerFactory.getLogger(CmdPatchApp::class.java) diff --git a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/general/CommandPatchApp.kt b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/general/CommandPatchApp.kt index 0b2da65c..fec3027a 100644 --- a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/general/CommandPatchApp.kt +++ b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/general/CommandPatchApp.kt @@ -2,6 +2,7 @@ package com.simiacryptus.skyenet.apps.general import com.simiacryptus.diff.FileValidationUtils import com.simiacryptus.jopenai.ChatClient +import com.simiacryptus.jopenai.models.ChatModel import com.simiacryptus.jopenai.models.TextModel import com.simiacryptus.skyenet.core.platform.Session import com.simiacryptus.skyenet.webui.session.SessionTask @@ -13,7 +14,7 @@ class CommandPatchApp( session: Session, settings: Settings, api: ChatClient, - model: TextModel, + model: ChatModel, private val files: Array?, val command: String, ) : PatchApp(root, session, settings, api, model) { diff --git a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/general/PatchApp.kt b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/general/PatchApp.kt index 2828e49c..43799165 100644 --- a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/general/PatchApp.kt +++ b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/general/PatchApp.kt @@ -4,7 +4,7 @@ import com.simiacryptus.diff.FileValidationUtils import com.simiacryptus.diff.addApplyFileDiffLinks import com.simiacryptus.jopenai.ChatClient import com.simiacryptus.jopenai.describe.Description -import com.simiacryptus.jopenai.models.TextModel +import com.simiacryptus.jopenai.models.ChatModel import com.simiacryptus.skyenet.AgentPatterns import com.simiacryptus.skyenet.Retryable import com.simiacryptus.skyenet.core.actors.ParsedActor @@ -21,13 +21,14 @@ import com.simiacryptus.util.JsonUtil import org.slf4j.LoggerFactory import java.io.File import java.nio.file.Path +import java.util.UUID abstract class PatchApp( override val root: File, val session: Session, val settings: Settings, val api: ChatClient, - val model: TextModel, + val model: ChatModel, val promptPrefix: String = """The following command was run and produced an error:""" ) : ApplicationServer( applicationName = "Magic Code Fixer", @@ -177,8 +178,23 @@ abstract class PatchApp( changed: MutableSet, api: ChatClient, ) { + val api = api.getChildClient().apply { + val createFile = task.createFile(".logs/api-${UUID.randomUUID()}.log") + createFile.second?.apply { + logStreams += this.outputStream().buffered() + task.verbose("API log: $this") + } + } val plan = ParsedActor( resultClass = ParsedErrors::class.java, + exampleInstance = ParsedErrors(listOf( + ParsedError( + message = "Error message", + relatedFiles = listOf("src/main/java/com/example/Example.java"), + fixFiles = listOf("src/main/java/com/example/Example.java"), + searchStrings = listOf("def exampleFunction", "TODO") + ) + )), prompt = """ |You are a helpful AI that helps people with coding. | @@ -349,7 +365,8 @@ abstract class PatchApp( } else { false } - } + }, + model = model, ) content.clear() content.append("
${MarkdownUtil.renderMarkdown(markdown!!)}
") diff --git a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/general/PlanAheadApp.kt b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/general/PlanAheadApp.kt index f99db7a5..58b1f092 100644 --- a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/general/PlanAheadApp.kt +++ b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/general/PlanAheadApp.kt @@ -2,6 +2,7 @@ package com.simiacryptus.skyenet.apps.general import com.simiacryptus.jopenai.API import com.simiacryptus.jopenai.ChatClient +import com.simiacryptus.jopenai.models.ChatModel import com.simiacryptus.jopenai.models.TextModel import com.simiacryptus.skyenet.apps.plan.PlanCoordinator import com.simiacryptus.skyenet.apps.plan.PlanCoordinator.Companion.initialPlan @@ -22,8 +23,8 @@ open class PlanAheadApp( applicationName: String = "Task Planning v1.1", path: String = "/taskDev", val planSettings: PlanSettings, - val model: TextModel, - val parsingModel: TextModel, + val model: ChatModel, + val parsingModel: ChatModel, val domainName: String = "localhost", showMenubar: Boolean = true, val initialPlan: TaskBreakdownWithPrompt? = null, diff --git a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/general/PlanChatApp.kt b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/general/PlanChatApp.kt index 9a3c990a..086826c8 100644 --- a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/general/PlanChatApp.kt +++ b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/general/PlanChatApp.kt @@ -2,6 +2,7 @@ package com.simiacryptus.skyenet.apps.general import com.simiacryptus.jopenai.API import com.simiacryptus.jopenai.ChatClient +import com.simiacryptus.jopenai.models.ChatModel import com.simiacryptus.jopenai.models.TextModel import com.simiacryptus.skyenet.apps.plan.* import com.simiacryptus.skyenet.apps.plan.file.InquiryTask.InquiryTaskData @@ -17,8 +18,8 @@ open class PlanChatApp( applicationName: String = "Task Planning Chat v1.0", path: String = "/taskChat", planSettings: PlanSettings, - model: TextModel, - parsingModel: TextModel, + model: ChatModel, + parsingModel: ChatModel, domainName: String = "localhost", showMenubar: Boolean = true, initialPlan: TaskBreakdownWithPrompt? = null, @@ -77,7 +78,10 @@ open class PlanChatApp( command = planSettings.command, temperature = planSettings.temperature, workingDir = planSettings.workingDir, - env = planSettings.env + env = planSettings.env, + githubToken = planSettings.githubToken, + googleApiKey = planSettings.googleApiKey, + googleSearchEngineId = planSettings.googleSearchEngineId, )).copy( allowBlocking = false, ) diff --git a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/general/WebDevApp.kt b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/general/WebDevApp.kt index 2ac65f1e..cfe33195 100644 --- a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/general/WebDevApp.kt +++ b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/general/WebDevApp.kt @@ -375,7 +375,8 @@ class WebDevAgent( } }, ui = ui, - api = api + api = api, + model = model, ) ) }, diff --git a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/AbstractTask.kt b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/AbstractTask.kt index 98d18bc0..da8ca8ac 100644 --- a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/AbstractTask.kt +++ b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/AbstractTask.kt @@ -54,7 +54,6 @@ abstract class AbstractTask( abstract fun run( agent: PlanCoordinator, - taskId: String, messages: List = listOf(), task: SessionTask, api: API, diff --git a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/CommandAutoFixTask.kt b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/CommandAutoFixTask.kt index 9ce68b2f..0ec6a4fd 100644 --- a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/CommandAutoFixTask.kt +++ b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/CommandAutoFixTask.kt @@ -35,7 +35,7 @@ class CommandAutoFixTask( data class CommandWithWorkingDir( @Description("The command to be executed") val command: List, - @Description("The working directory for this specific command") + @Description("The relative path of the working directory") val workingDir: String? = null ) @@ -53,7 +53,6 @@ ${planSettings.commandAutoFixCommands?.joinToString("\n") { " * ${File(it).na override fun run( agent: PlanCoordinator, - taskId: String, messages: List, task: SessionTask, api: API, @@ -140,7 +139,6 @@ ${planSettings.commandAutoFixCommands?.joinToString("\n") { " * ${File(it).na } catch (e: Throwable) { log.warn("Error", e) } - log.debug("Completed command auto fix: $taskId") } companion object { diff --git a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/ForeachTask.kt b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/ForeachTask.kt index 866e1656..71a539a0 100644 --- a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/ForeachTask.kt +++ b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/ForeachTask.kt @@ -38,7 +38,6 @@ ForeachTask - Execute a task for each item in a list override fun run( agent: PlanCoordinator, - taskId: String, messages: List, task: SessionTask, api: API, diff --git a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/PlanCoordinator.kt b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/PlanCoordinator.kt index ab8812af..8e74c914 100644 --- a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/PlanCoordinator.kt +++ b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/PlanCoordinator.kt @@ -22,6 +22,7 @@ import org.slf4j.LoggerFactory import java.io.File import java.nio.file.Path import java.util.* +import java.util.concurrent.Future import java.util.concurrent.ThreadPoolExecutor import java.util.concurrent.TimeUnit @@ -232,7 +233,7 @@ class PlanCoordinator( """.trimMargin(), ui = ui ) ) - val api = (api as ChatClient).getChildClient().apply { + val api = api.getChildClient().apply { val createFile = task1.createFile(".logs/api-${UUID.randomUUID()}.log") createFile.second?.apply { logStreams += this.outputStream().buffered() @@ -247,7 +248,6 @@ class PlanCoordinator( ) impl.run( agent = this, - taskId = taskId, messages = messages, task = task1, api = api, @@ -264,16 +264,13 @@ class PlanCoordinator( } } } + await(planProcessingState.taskFutures) + } + + fun await(futures: MutableMap>) { val start = System.currentTimeMillis() - planProcessingState.taskFutures.forEach { (id, future) -> - try { - future.get( - (TimeUnit.MINUTES.toMillis(1) - (System.currentTimeMillis() - start)).coerceAtLeast(0), - TimeUnit.MILLISECONDS - ) ?: log.warn("Dependency not found: $id") - } catch (e: Throwable) { - log.warn("Error", e) - } + while (futures.values.count { it.isDone } < futures.size && (System.currentTimeMillis() - start) < TimeUnit.MINUTES.toMillis(2)) { + Thread.sleep(1000) } } diff --git a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/PlanSettings.kt b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/PlanSettings.kt index fa2218fe..5c457d73 100644 --- a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/PlanSettings.kt +++ b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/PlanSettings.kt @@ -1,6 +1,7 @@ package com.simiacryptus.skyenet.apps.plan import com.simiacryptus.jopenai.describe.AbbrevWhitelistYamlDescriber +import com.simiacryptus.jopenai.models.ChatModel import com.simiacryptus.jopenai.models.TextModel import com.simiacryptus.skyenet.apps.plan.CommandAutoFixTask.CommandAutoFixTaskData import com.simiacryptus.skyenet.apps.plan.PlanUtil.isWindows @@ -14,12 +15,12 @@ import com.simiacryptus.skyenet.core.actors.ParsedActor data class TaskSettings( var enabled: Boolean = false, - var model: TextModel? = null + var model: ChatModel? = null ) open class PlanSettings( - var defaultModel: TextModel, - var parsingModel: TextModel, + var defaultModel: ChatModel, + var parsingModel: ChatModel, val command: List = listOf(if (isWindows) "powershell" else "bash"), var temperature: Double = 0.2, val budget: Double = 2.0, @@ -37,6 +38,9 @@ open class PlanSettings( val env: Map? = mapOf(), val workingDir: String? = ".", val language: String? = if (isWindows) "powershell" else "bash", + var githubToken: String? = null, + var googleApiKey: String? = null, + var googleSearchEngineId: String? = null, ) { fun getTaskSettings(taskType: TaskType<*>): TaskSettings = @@ -47,8 +51,8 @@ open class PlanSettings( } fun copy( - model: TextModel = this.defaultModel, - parsingModel: TextModel = this.parsingModel, + model: ChatModel = this.defaultModel, + parsingModel: ChatModel = this.parsingModel, command: List = this.command, temperature: Double = this.temperature, budget: Double = this.budget, @@ -72,6 +76,9 @@ open class PlanSettings( env = env, workingDir = workingDir, language = language, + githubToken = this.githubToken, + googleApiKey = this.googleApiKey, + googleSearchEngineId = this.googleSearchEngineId, ) fun planningActor(): ParsedActor { diff --git a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/PlanningTask.kt b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/PlanningTask.kt index 94de30e9..70c7cc50 100644 --- a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/PlanningTask.kt +++ b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/PlanningTask.kt @@ -54,7 +54,6 @@ class PlanningTask( override fun run( agent: PlanCoordinator, - taskId: String, messages: List, task: SessionTask, api: API, diff --git a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/RunShellCommandTask.kt b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/RunShellCommandTask.kt index 3249ff96..cc2561d4 100644 --- a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/RunShellCommandTask.kt +++ b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/RunShellCommandTask.kt @@ -21,7 +21,7 @@ class RunShellCommandTask( class RunShellCommandTaskData( @Description("The shell command to be executed") val command: String? = null, - @Description("The working directory for the command execution") + @Description("The relative file path of the working directory") val workingDir: String? = null, task_description: String? = null, task_dependencies: List? = null, @@ -66,7 +66,6 @@ Note: This task is for running simple and safe commands. Avoid executing command override fun run( agent: PlanCoordinator, - taskId: String, messages: List, task: SessionTask, api: API, @@ -140,7 +139,6 @@ Note: This task is for running simple and safe commands. Avoid executing command } catch (e: Throwable) { log.warn("Error", e) } - log.debug("Completed shell command: $taskId") } companion object { diff --git a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/TaskType.kt b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/TaskType.kt index 7d35a4bb..ea13a3e4 100644 --- a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/TaskType.kt +++ b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/TaskType.kt @@ -17,6 +17,7 @@ import com.simiacryptus.skyenet.apps.plan.file.CodeOptimizationTask.CodeOptimiza import com.simiacryptus.skyenet.apps.plan.file.CodeReviewTask.CodeReviewTaskData import com.simiacryptus.skyenet.apps.plan.file.DocumentationTask.DocumentationTaskData import com.simiacryptus.skyenet.apps.plan.file.FileModificationTask.FileModificationTaskData +import com.simiacryptus.skyenet.apps.plan.file.GoogleSearchTask.GoogleSearchTaskData import com.simiacryptus.skyenet.apps.plan.file.InquiryTask.InquiryTaskData import com.simiacryptus.skyenet.apps.plan.file.PerformanceAnalysisTask.PerformanceAnalysisTaskData import com.simiacryptus.skyenet.apps.plan.file.RefactorTask.RefactorTaskData @@ -52,6 +53,9 @@ class TaskType( val RunShellCommand = TaskType("RunShellCommand", RunShellCommandTaskData::class.java) val CommandAutoFix = TaskType("CommandAutoFix", CommandAutoFixTaskData::class.java) val ForeachTask = TaskType("ForeachTask", ForeachTaskData::class.java) + val GitHubSearch = TaskType("GitHubSearch", GitHubSearchTask.GitHubSearchTaskData::class.java) + val GoogleSearch = TaskType("GoogleSearch", GoogleSearchTaskData::class.java) + val WebFetchAndTransform = TaskType("WebFetchAndTransform", WebFetchAndTransformTask.WebFetchAndTransformTaskData::class.java) init { registerConstructor(CommandAutoFix) { settings, task -> CommandAutoFixTask(settings, task) } @@ -69,6 +73,9 @@ class TaskType( registerConstructor(RefactorTask) { settings, task -> RefactorTask(settings, task) } registerConstructor(ForeachTask) { settings, task -> ForeachTask(settings, task) } registerConstructor(TaskPlanning) { settings, task -> PlanningTask(settings, task) } + registerConstructor(GitHubSearch) { settings, task -> GitHubSearchTask(settings, task) } + registerConstructor(GoogleSearch) { settings, task -> GoogleSearchTask(settings, task) } + registerConstructor(WebFetchAndTransform) { settings, task -> WebFetchAndTransformTask(settings, task) } } private fun registerConstructor( diff --git a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/file/AbstractAnalysisTask.kt b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/file/AbstractAnalysisTask.kt index 800634fd..98f50877 100644 --- a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/file/AbstractAnalysisTask.kt +++ b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/file/AbstractAnalysisTask.kt @@ -30,7 +30,6 @@ abstract class AbstractAnalysisTask( override fun run( agent: PlanCoordinator, - taskId: String, messages: List, task: SessionTask, api: API, diff --git a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/file/AbstractFileTask.kt b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/file/AbstractFileTask.kt index f273b0c4..fda5699a 100644 --- a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/file/AbstractFileTask.kt +++ b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/file/AbstractFileTask.kt @@ -19,9 +19,9 @@ abstract class AbstractFileTask( task_type: String, task_description: String? = null, task_dependencies: List? = null, - @Description("The file names to be used as input for the task") + @Description("The relative file paths to be used as input for the task") val input_files: List? = null, - @Description("The file names to be generated as output for the task") + @Description("The relative file paths to be generated as output for the task") val output_files: List? = null, state: TaskState? = TaskState.Pending, ) : PlanTaskBase( diff --git a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/file/DocumentationTask.kt b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/file/DocumentationTask.kt index 3230390a..ae3740df 100644 --- a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/file/DocumentationTask.kt +++ b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/file/DocumentationTask.kt @@ -3,6 +3,7 @@ package com.simiacryptus.skyenet.apps.plan.file import com.simiacryptus.diff.addApplyFileDiffLinks import com.simiacryptus.jopenai.API import com.simiacryptus.jopenai.describe.Description +import com.simiacryptus.jopenai.models.chatModel import com.simiacryptus.skyenet.Retryable import com.simiacryptus.skyenet.apps.plan.* import com.simiacryptus.skyenet.apps.plan.file.DocumentationTask.DocumentationTaskData @@ -66,7 +67,6 @@ class DocumentationTask( override fun run( agent: PlanCoordinator, - taskId: String, messages: List, task: SessionTask, api: API, @@ -101,7 +101,8 @@ class DocumentationTask( }, ui = agent.ui, api = api, - shouldAutoApply = { agent.planSettings.autoFix } + shouldAutoApply = { agent.planSettings.autoFix }, + model = planSettings.getTaskSettings(TaskType.Documentation).model ?: planSettings.defaultModel, ) task.complete() onComplete() @@ -117,7 +118,8 @@ class DocumentationTask( } }, ui = agent.ui, - api = api + api = api, + model = planSettings.getTaskSettings(TaskType.Documentation).model ?: planSettings.defaultModel, ) + acceptButtonFooter(agent.ui) { task.complete() onComplete() diff --git a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/file/EmbeddingSearchTask.kt b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/file/EmbeddingSearchTask.kt index e44e5069..b00cf31d 100644 --- a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/file/EmbeddingSearchTask.kt +++ b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/file/EmbeddingSearchTask.kt @@ -47,7 +47,6 @@ EmbeddingSearch - Search for similar embeddings in index files and provide top r override fun run( agent: PlanCoordinator, - taskId: String, messages: List, task: SessionTask, api: API, diff --git a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/file/FileModificationTask.kt b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/file/FileModificationTask.kt index 0dcacc9b..dbf99a58 100644 --- a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/file/FileModificationTask.kt +++ b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/file/FileModificationTask.kt @@ -7,7 +7,7 @@ import com.simiacryptus.skyenet.Retryable import com.simiacryptus.skyenet.apps.plan.* import com.simiacryptus.skyenet.apps.plan.file.FileModificationTask.FileModificationTaskData import com.simiacryptus.skyenet.core.actors.SimpleActor -import com.simiacryptus.skyenet.util.MarkdownUtil +import com.simiacryptus.skyenet.util.MarkdownUtil.renderMarkdown import com.simiacryptus.skyenet.webui.session.SessionTask import org.slf4j.LoggerFactory import java.util.concurrent.Semaphore @@ -17,12 +17,10 @@ class FileModificationTask( planTask: FileModificationTaskData? ) : AbstractFileTask(planSettings, planTask) { class FileModificationTaskData( - @Description("List of input files to be examined when designing the modifications") input_files: List? = null, - @Description("List of output files to be modified or created") output_files: List? = null, @Description("Specific modifications to be made to the files") - val modifications: Map? = null, + val modifications: Any? = null, task_description: String? = null, task_dependencies: List? = null, state: TaskState? = null @@ -97,14 +95,14 @@ class FileModificationTask( override fun run( agent: PlanCoordinator, - taskId: String, messages: List, task: SessionTask, api: API, resultFn: (String) -> Unit ) { if (((planTask?.input_files ?: listOf()) + (planTask?.output_files ?: listOf())).isEmpty()) { - task.complete("No input files specified") + task.complete("CONFIGURATION ERROR: No input files specified") + resultFn("CONFIGURATION ERROR: No input files specified") return } val semaphore = Semaphore(0) @@ -128,13 +126,14 @@ class FileModificationTask( }, ui = agent.ui, api = api, - shouldAutoApply = { agent.planSettings.autoFix } + shouldAutoApply = { agent.planSettings.autoFix }, + model = planSettings.getTaskSettings(TaskType.FileModification).model ?: planSettings.defaultModel, ) task.complete() onComplete() - MarkdownUtil.renderMarkdown(diffLinks + "\n\n## Auto-applied changes", ui = agent.ui) + renderMarkdown(diffLinks + "\n\n## Auto-applied changes", ui = agent.ui) } else { - MarkdownUtil.renderMarkdown( + renderMarkdown( agent.ui.socketManager!!.addApplyFileDiffLinks( root = agent.root, response = codeResult, @@ -144,7 +143,8 @@ class FileModificationTask( } }, ui = agent.ui, - api = api + api = api, + model = planSettings.getTaskSettings(TaskType.FileModification).model ?: planSettings.defaultModel, ) + acceptButtonFooter(agent.ui) { task.complete() onComplete() diff --git a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/file/GitHubSearchTask.kt b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/file/GitHubSearchTask.kt new file mode 100644 index 00000000..cdc700e9 --- /dev/null +++ b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/file/GitHubSearchTask.kt @@ -0,0 +1,153 @@ +package com.simiacryptus.skyenet.apps.plan.file + +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.module.kotlin.readValue +import com.simiacryptus.jopenai.API +import com.simiacryptus.jopenai.describe.Description +import com.simiacryptus.skyenet.apps.plan.* +import com.simiacryptus.skyenet.util.MarkdownUtil +import com.simiacryptus.skyenet.webui.session.SessionTask +import org.slf4j.LoggerFactory +import java.net.URI +import java.net.http.HttpClient +import java.net.http.HttpRequest +import java.net.http.HttpResponse + +class GitHubSearchTask( + planSettings: PlanSettings, + planTask: GitHubSearchTaskData? +) : AbstractTask(planSettings, planTask) { + class GitHubSearchTaskData( + @Description("The search query to use for GitHub search") + val search_query: String, + @Description("The type of GitHub search to perform (code, commits, issues, repositories, topics, users)") + val search_type: String, + @Description("The number of results to return (max 100)") + val per_page: Int = 30, + @Description("Sort order for results") + val sort: String? = null, + @Description("Sort direction (asc or desc)") + val order: String? = null, + task_description: String? = null, + task_dependencies: List? = null, + state: TaskState? = null, + ) : PlanTaskBase( + task_type = TaskType.GitHubSearch.name, + task_description = task_description, + task_dependencies = task_dependencies, + state = state + ) + + override fun promptSegment() = """ +GitHubSearch - Search GitHub for code, commits, issues, repositories, topics, or users +** Specify the search query +** Specify the type of search (code, commits, issues, repositories, topics, users) +** Specify the number of results to return (max 100) +** Optionally specify sort order (e.g. stars, forks, updated) +** Optionally specify sort direction (asc or desc) +""".trimMargin() + + override fun run( + agent: PlanCoordinator, + messages: List, + task: SessionTask, + api: API, + resultFn: (String) -> Unit + ) { + val searchResults = performGitHubSearch() + val formattedResults = formatSearchResults(searchResults) + task.add(MarkdownUtil.renderMarkdown(formattedResults, ui = agent.ui)) + resultFn(formattedResults) + } + + private fun performGitHubSearch(): String { + val client = HttpClient.newBuilder().build() + val uriBuilder = StringBuilder("https://api.github.com/search/${planTask?.search_type}?q=${planTask?.search_query}&per_page=${planTask?.per_page}") + planTask?.sort?.let { uriBuilder.append("&sort=$it") } + planTask?.order?.let { uriBuilder.append("&order=$it") } + val request = HttpRequest.newBuilder() + .uri(URI.create(uriBuilder.toString())) + .header("Accept", "application/vnd.github+json") + .header("Authorization", "Bearer ${planSettings.githubToken}") + .header("X-GitHub-Api-Version", "2022-11-28") + .GET() + .build() + + val response = client.send(request, HttpResponse.BodyHandlers.ofString()) + if (response.statusCode() != 200) { + throw RuntimeException("GitHub API request failed with status ${response.statusCode()}: ${response.body()}") + } + return response.body() + } + + private fun formatSearchResults(results: String): String { + val mapper = ObjectMapper() + val searchResults: Map = mapper.readValue(results) + return buildString { + appendLine("# GitHub Search Results") + appendLine() + appendLine("Total results: ${searchResults["total_count"]}") + appendLine() + appendLine("## Top Results:") + appendLine() + val items = searchResults["items"] as List> + items.take(10).forEach { item -> + when (planTask?.search_type) { + "repositories" -> formatRepositoryResult(item) + "code" -> formatCodeResult(item) + "commits" -> formatCommitResult(item) + "issues" -> formatIssueResult(item) + "users" -> formatUserResult(item) + "topics" -> formatTopicResult(item) + else -> appendLine("- ${item["name"] ?: item["title"] ?: item["login"]}") + } + appendLine() + } + } + } + + private fun StringBuilder.formatTopicResult(topic: Map) { + appendLine("### [${topic["name"]}](${topic["url"]})") + appendLine("${topic["short_description"]}") + appendLine("Featured: ${topic["featured"]} | Curated: ${topic["curated"]}") + } + + private fun StringBuilder.formatRepositoryResult(repo: Map) { + appendLine("### ${repo["full_name"]}") + appendLine("${repo["description"]}") + appendLine("Stars: ${repo["stargazers_count"]} | Forks: ${repo["forks_count"]}") + appendLine("[View on GitHub](${repo["html_url"]})") + } + + private fun StringBuilder.formatCodeResult(code: Map) { + val repo = code["repository"] as Map + appendLine("### [${repo["full_name"]}](${code["html_url"]})") + appendLine("File: ${code["path"]}") + appendLine("```") + appendLine(code["text_matches"]?.toString()?.take(200) ?: "") + appendLine("```") + } + + private fun StringBuilder.formatCommitResult(commit: Map) { + val repo = commit["repository"] as Map + appendLine("### [${repo["full_name"]}](${commit["html_url"]})") + appendLine("${(commit["commit"] as Map)["message"]}") + appendLine("Author: ${(commit["author"] as Map)["login"]} | Date: ${((commit["commit"] as Map)["author"] as Map)["date"]}") + } + + private fun StringBuilder.formatIssueResult(issue: Map) { + appendLine("### [${issue["title"]}](${issue["html_url"]})") + appendLine("State: ${issue["state"]} | Comments: ${issue["comments"]}") + appendLine("Created by ${(issue["user"] as Map)["login"]} on ${issue["created_at"]}") + } + + private fun StringBuilder.formatUserResult(user: Map) { + appendLine("### [${user["login"]}](${user["html_url"]})") + appendLine("Type: ${user["type"]} | Repos: ${user["public_repos"]}") + appendLine("![Avatar](${user["avatar_url"]})") + } + + companion object { + private val log = LoggerFactory.getLogger(GitHubSearchTask::class.java) + } +} \ No newline at end of file diff --git a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/file/GoogleSearchTask.kt b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/file/GoogleSearchTask.kt new file mode 100644 index 00000000..9c7cb29c --- /dev/null +++ b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/file/GoogleSearchTask.kt @@ -0,0 +1,95 @@ +package com.simiacryptus.skyenet.apps.plan.file + +import com.simiacryptus.jopenai.API +import com.simiacryptus.jopenai.describe.Description +import com.simiacryptus.skyenet.apps.plan.* +import com.simiacryptus.skyenet.util.MarkdownUtil +import com.simiacryptus.skyenet.webui.session.SessionTask +import org.slf4j.LoggerFactory +import java.net.URI +import java.net.URLEncoder +import java.net.http.HttpClient +import java.net.http.HttpRequest +import java.net.http.HttpResponse +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.module.kotlin.readValue +import com.simiacryptus.util.JsonUtil + +class GoogleSearchTask( + planSettings: PlanSettings, + planTask: GoogleSearchTaskData? +) : AbstractTask(planSettings, planTask) { + class GoogleSearchTaskData( + @Description("The search query to use for Google search") + val search_query: String, + @Description("The number of results to return (max 10)") + val num_results: Int = 5, + task_description: String? = null, + task_dependencies: List? = null, + state: TaskState? = null, + ) : PlanTaskBase( + task_type = TaskType.GoogleSearch.name, + task_description = task_description, + task_dependencies = task_dependencies, + state = state + ) + + override fun promptSegment() = """ +GoogleSearch - Search Google for web results +** Specify the search query +** Specify the number of results to return (max 10) +""".trimMargin() + + override fun run( + agent: PlanCoordinator, + messages: List, + task: SessionTask, + api: API, + resultFn: (String) -> Unit + ) { + val searchResults = performGoogleSearch() + val formattedResults = formatSearchResults(searchResults) + task.add(MarkdownUtil.renderMarkdown(formattedResults, ui = agent.ui)) + resultFn(formattedResults) + } + + private fun performGoogleSearch(): String { + val client = HttpClient.newBuilder().build() + val encodedQuery = URLEncoder.encode(planTask?.search_query, "UTF-8") + val uriBuilder = "https://www.googleapis.com/customsearch/v1?key=${planSettings.googleApiKey}&cx=${planSettings.googleSearchEngineId}&q=$encodedQuery&num=${planTask?.num_results}" + + val request = HttpRequest.newBuilder() + .uri(URI.create(uriBuilder)) + .GET() + .build() + + val response = client.send(request, HttpResponse.BodyHandlers.ofString()) + if (response.statusCode() != 200) { + throw RuntimeException("Google API request failed with status ${response.statusCode()}: ${response.body()}") + } + return response.body() + } + + private fun formatSearchResults(results: String): String { + val mapper = ObjectMapper() + val searchResults: Map = mapper.readValue(results) + return buildString { + appendLine("# Google Search Results") + appendLine() + val items = searchResults["items"] as List>? + items?.forEachIndexed { index, item -> + appendLine("# ${index + 1}. [${item["title"]}](${item["link"]})") + appendLine("${item["htmlSnippet"]}") + appendLine("Pagemap:") + appendLine("```json") + appendLine(JsonUtil.toJson(item["pagemap"] ?: "")) + appendLine("```") + appendLine() + } ?: appendLine("No results found.") + } + } + + companion object { + private val log = LoggerFactory.getLogger(GoogleSearchTask::class.java) + } +} \ No newline at end of file diff --git a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/file/InquiryTask.kt b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/file/InquiryTask.kt index d574ecd4..8ed4899b 100644 --- a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/file/InquiryTask.kt +++ b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/file/InquiryTask.kt @@ -73,7 +73,6 @@ class InquiryTask( override fun run( agent: PlanCoordinator, - taskId: String, messages: List, task: SessionTask, api: API, diff --git a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/file/SearchTask.kt b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/file/SearchTask.kt index 757f93b9..96200c8f 100644 --- a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/file/SearchTask.kt +++ b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/file/SearchTask.kt @@ -45,7 +45,6 @@ Search - Search for patterns in files and provide results with context override fun run( agent: PlanCoordinator, - taskId: String, messages: List, task: SessionTask, api: API, diff --git a/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/file/WebFetchAndTransformTask.kt b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/file/WebFetchAndTransformTask.kt new file mode 100644 index 00000000..cac9445b --- /dev/null +++ b/webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/file/WebFetchAndTransformTask.kt @@ -0,0 +1,141 @@ +package com.simiacryptus.skyenet.apps.plan.file + +import com.simiacryptus.jopenai.API +import com.simiacryptus.jopenai.describe.Description +import com.simiacryptus.skyenet.apps.plan.* +import com.simiacryptus.skyenet.core.actors.SimpleActor +import com.simiacryptus.skyenet.util.MarkdownUtil +import com.simiacryptus.skyenet.webui.session.SessionTask +import org.apache.hc.client5.http.classic.methods.HttpGet +import org.apache.hc.client5.http.impl.classic.HttpClients +import org.apache.hc.core5.http.io.entity.EntityUtils +import org.jsoup.Jsoup +import org.jsoup.nodes.Document +import org.slf4j.LoggerFactory + +open class WebFetchAndTransformTask( + planSettings: PlanSettings, + planTask: WebFetchAndTransformTaskData? +) : AbstractTask(planSettings, planTask) { + class WebFetchAndTransformTaskData( + @Description("The URL to fetch") + val url: String, + @Description("The desired format or focus for the transformation") + val transformationGoal: String, + task_description: String? = null, + task_dependencies: List? = null, + state: TaskState? = null, + ) : PlanTaskBase( + task_type = TaskType.WebFetchAndTransform.name, + task_description = task_description, + task_dependencies = task_dependencies, + state = state + ) + + override fun promptSegment() = """ + WebFetchAndTransform - Fetch a web page, strip HTML, and transform content + ** Specify the URL to fetch + ** Specify the desired format or focus for the transformation + """.trimMargin() + + override fun run( + agent: PlanCoordinator, + messages: List, + task: SessionTask, + api: API, + resultFn: (String) -> Unit + ) { + val fetchedContent = fetchAndStripHtml(planTask?.url ?: "") + val transformedContent = transformContent(fetchedContent, planTask?.transformationGoal ?: "", api) + task.add(MarkdownUtil.renderMarkdown(transformedContent, ui = agent.ui)) + resultFn(transformedContent) + } + + private fun fetchAndStripHtml(url: String): String { + HttpClients.createDefault().use { httpClient -> + val httpGet = HttpGet(url) + httpClient.execute(httpGet).use { response -> + val entity = response.entity + val content = EntityUtils.toString(entity) + return scrubHtml(content) + } + } + } + + fun scrubHtml(str: String, maxLength: Int = 100 * 1024): String { + val document: Document = Jsoup.parse(str) + // Remove unnecessary elements, attributes, and optimize the document + document.apply { + if (document.body().html().length > maxLength) return@apply + select("script, style, link, meta, iframe, noscript").remove() // Remove unnecessary and potentially harmful tags + outputSettings().prettyPrint(false) // Disable pretty printing for compact output + if (document.body().html().length > maxLength) return@apply + // Remove comments + select("*").forEach { it.childNodes().removeAll { node -> node.nodeName() == "#comment" } } + if (document.body().html().length > maxLength) return@apply + // Remove data-* attributes + select("*[data-*]").forEach { it.attributes().removeAll { attr -> attr.key.startsWith("data-") } } + if (document.body().html().length > maxLength) return@apply + select("*").forEach { element -> + val importantAttributes = setOf("href", "src", "alt", "title", "width", "height", "style", "class", "id", "name") + element.attributes().removeAll { it.key !in importantAttributes } + } + if (document.body().html().length > maxLength) return@apply + // Remove empty elements + select("*").filter { it.text().isBlank() && it.attributes().isEmpty() && !it.hasAttr("img") }.forEach { remove() } + if (document.body().html().length > maxLength) return@apply + // Unwrap single-child elements with no attributes + select("*").forEach { element -> + if (element.childNodes().size == 1 && element.childNodes()[0].nodeName() == "#text" && element.attributes().isEmpty()) { + element.unwrap() + } + } + if (document.body().html().length > maxLength) return@apply + // Convert relative URLs to absolute + select("[href],[src]").forEach { element -> + element.attr("href").let { href -> element.attr("href", href) } + element.attr("src").let { src -> element.attr("src", src) } + } + if (document.body().html().length > maxLength) return@apply + // Remove empty attributes + select("*").forEach { element -> + element.attributes().removeAll { it.value.isBlank() } + } + } + + // Truncate if necessary + val result = document.body().html() + return if (result.length > maxLength) { + result.substring(0, maxLength) + } else { + result + } + } + + private fun transformContent(content: String, transformationGoal: String, api: API): String { + val prompt = """ + Transform the following web content according to this goal: $transformationGoal + + Content: + $content + + Transformed content: + """.trimIndent() + return SimpleActor( + prompt = prompt, + model = planSettings.defaultModel, + ).answer( + listOf( + """ + |Transform the following web content according to this goal: $transformationGoal + | + |$content + """.trimMargin(), + ), api + ) + } + + companion object { + private val log = LoggerFactory.getLogger(WebFetchAndTransformTask::class.java) + } +} \ No newline at end of file diff --git a/webui/src/main/kotlin/com/simiacryptus/skyenet/webui/test/FilePatchTestApp.kt b/webui/src/main/kotlin/com/simiacryptus/skyenet/webui/test/FilePatchTestApp.kt index a827579c..d45216b7 100644 --- a/webui/src/main/kotlin/com/simiacryptus/skyenet/webui/test/FilePatchTestApp.kt +++ b/webui/src/main/kotlin/com/simiacryptus/skyenet/webui/test/FilePatchTestApp.kt @@ -34,7 +34,6 @@ open class FilePatchTestApp( val sourceFile = Files.createTempFile("source", ".txt").toFile() sourceFile.writeText(source) sourceFile.deleteOnExit() - //Desktop.getDesktop().open(sourceFile) val patch = """ |# ${sourceFile.name} @@ -44,7 +43,12 @@ open class FilePatchTestApp( |+Goodbye, World! |``` """.trimMargin() - val newPatch = socketManager.addApplyFileDiffLinks(sourceFile.toPath().parent, patch, {}, ui, api) + val newPatch = socketManager.addApplyFileDiffLinks( + root = sourceFile.toPath().parent, + response = patch, + ui = ui, + api = api + ) task.complete(renderMarkdown(newPatch, ui = ui)) return socketManager