From 2555afdfee939db1c73e2935dcdb1160453e4efa Mon Sep 17 00:00:00 2001 From: Simon Vergauwen Date: Tue, 9 Jul 2024 09:42:30 +0200 Subject: [PATCH] Deprecate, and remove SQL (#767) --- .../functional/xef/llm/models/OpenAIClient.kt | 1 + examples/build.gradle.kts | 1 - integrations/sql/build.gradle.kts | 45 ----- .../xebia/functional/xef/sql/DatabaseOps.kt | 37 ---- .../xebia/functional/xef/sql/QueryResult.kt | 22 --- .../xebia/functional/xef/sql/ResultSetOps.kt | 35 ---- .../com/xebia/functional/xef/sql/SQL.kt | 160 ------------------ .../functional/xef/sql/jdbc/JdbcConfig.kt | 17 -- settings.gradle.kts | 3 - 9 files changed, 1 insertion(+), 320 deletions(-) delete mode 100644 integrations/sql/build.gradle.kts delete mode 100644 integrations/sql/src/main/kotlin/com/xebia/functional/xef/sql/DatabaseOps.kt delete mode 100644 integrations/sql/src/main/kotlin/com/xebia/functional/xef/sql/QueryResult.kt delete mode 100644 integrations/sql/src/main/kotlin/com/xebia/functional/xef/sql/ResultSetOps.kt delete mode 100644 integrations/sql/src/main/kotlin/com/xebia/functional/xef/sql/SQL.kt delete mode 100644 integrations/sql/src/main/kotlin/com/xebia/functional/xef/sql/jdbc/JdbcConfig.kt diff --git a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/models/OpenAIClient.kt b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/models/OpenAIClient.kt index ba9b824f5..d14851757 100644 --- a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/models/OpenAIClient.kt +++ b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/models/OpenAIClient.kt @@ -3,6 +3,7 @@ package com.xebia.functional.xef.llm.models import com.xebia.functional.openai.generated.model.CreateChatCompletionRequestModel import com.xebia.functional.openai.generated.model.CreateEmbeddingRequestModel import com.xebia.functional.tokenizer.ModelType +import io.ktor.client.* fun CreateChatCompletionRequestModel.modelType(forFunctions: Boolean = false): ModelType { val stringValue = value diff --git a/examples/build.gradle.kts b/examples/build.gradle.kts index b29a285e2..2a60c1ded 100644 --- a/examples/build.gradle.kts +++ b/examples/build.gradle.kts @@ -19,7 +19,6 @@ dependencies { implementation(projects.xefEvaluator) implementation(projects.xefFilesystem) implementation(projects.xefPdf) - implementation(projects.xefSql) implementation(projects.xefTokenizer) implementation(projects.xefReasoning) implementation(projects.xefOpentelemetry) diff --git a/integrations/sql/build.gradle.kts b/integrations/sql/build.gradle.kts deleted file mode 100644 index ec2b7129e..000000000 --- a/integrations/sql/build.gradle.kts +++ /dev/null @@ -1,45 +0,0 @@ -plugins { - id(libs.plugins.kotlin.jvm.get().pluginId) - alias(libs.plugins.arrow.gradle.publish) - alias(libs.plugins.semver.gradle) - alias(libs.plugins.detekt) - alias(libs.plugins.kotlinx.serialization) -} - -dependencies { detektPlugins(project(":detekt-rules")) } - -repositories { mavenCentral() } - -java { - sourceCompatibility = JavaVersion.VERSION_11 - targetCompatibility = JavaVersion.VERSION_11 - toolchain { languageVersion = JavaLanguageVersion.of(11) } -} - -detekt { - toolVersion = "1.23.1" - source.setFrom(files("src/main/kotlin")) - config.setFrom("../../config/detekt/detekt.yml") - autoCorrect = true -} - -dependencies { - implementation(projects.xefCore) - implementation(projects.xefTokenizer) - implementation(libs.klogging) - implementation(libs.exposed.core) - implementation(libs.exposed.dao) - implementation(libs.exposed.jdbc) -} - -tasks { - withType().configureEach { - dependsOn(":detekt-rules:assemble") - autoCorrect = true - } - named("detekt") { - dependsOn(":detekt-rules:assemble") - getByName("build").dependsOn(this) - } - withType { dependsOn(withType()) } -} diff --git a/integrations/sql/src/main/kotlin/com/xebia/functional/xef/sql/DatabaseOps.kt b/integrations/sql/src/main/kotlin/com/xebia/functional/xef/sql/DatabaseOps.kt deleted file mode 100644 index a6ff5ca05..000000000 --- a/integrations/sql/src/main/kotlin/com/xebia/functional/xef/sql/DatabaseOps.kt +++ /dev/null @@ -1,37 +0,0 @@ -package com.xebia.functional.xef.sql - -import org.jetbrains.exposed.sql.Table -import org.jetbrains.exposed.sql.exists -import org.jetbrains.exposed.sql.transactions.transaction -import java.sql.JDBCType - -fun tableDDL(tableNames: List): String = transaction { - tableNames.joinToString("\n") { tableName -> - val table = Table(tableName) - require(table.exists()) { "Table $tableName does not exist" } - - this.connection.metadata { - val columns = this.columns(table) - .values - .flatten() - .joinToString(",\n", postfix = ",") { column -> - val dataType = JDBCType.valueOf(column.type).name - val dataTypeWithLength = - (if (column.size != null) "$dataType(${column.size})" else dataType).uppercase() - val nullable = if (column.nullable) "NULL" else "NOT NULL" - val defaultValue = if (column.defaultDbValue != null) "DEFAULT ${column.defaultDbValue}" else "" - - "\t${column.name} $dataTypeWithLength $nullable $defaultValue" - }.replace(Regex("\\s*,"), ",") - - val primaryKeys = this.existingPrimaryKeys(table) - .values - .fold("") { _, pk -> pk?.columnNames?.joinToString { name -> "\tPRIMARY KEY ($name),\n" } ?: "" } - - """TABLE $tableName ( - |$columns - |$primaryKeys); - |""".trimMargin() - } - } -} diff --git a/integrations/sql/src/main/kotlin/com/xebia/functional/xef/sql/QueryResult.kt b/integrations/sql/src/main/kotlin/com/xebia/functional/xef/sql/QueryResult.kt deleted file mode 100644 index 6e010c724..000000000 --- a/integrations/sql/src/main/kotlin/com/xebia/functional/xef/sql/QueryResult.kt +++ /dev/null @@ -1,22 +0,0 @@ -package com.xebia.functional.xef.sql - -import kotlinx.serialization.Serializable - -@Serializable -data class QueryResult( - val columns: List, - val rows: List> -) { - companion object { - fun empty() = QueryResult(emptyList(), emptyList()) - } - - fun index(column: String): Int = - columns.indexOfFirst { it.name == column } -} - -@Serializable -data class Column( - val name: String, - val type: String -) diff --git a/integrations/sql/src/main/kotlin/com/xebia/functional/xef/sql/ResultSetOps.kt b/integrations/sql/src/main/kotlin/com/xebia/functional/xef/sql/ResultSetOps.kt deleted file mode 100644 index 794947a60..000000000 --- a/integrations/sql/src/main/kotlin/com/xebia/functional/xef/sql/ResultSetOps.kt +++ /dev/null @@ -1,35 +0,0 @@ -package com.xebia.functional.xef.sql - -import java.sql.ResultSet - -object ResultSetOps { - - fun ResultSet.toQueryResult(): QueryResult { - val columns = this.getColumns() - val rows = mutableListOf>() - - while (this.next()) { - val row = mutableListOf() - - for (i in 1..this.metaData.columnCount) { - row.add(getString(i)) - } - - rows.add(row) - } - - return QueryResult(columns, rows) - } - - private fun ResultSet.getColumns(): List { - val columns = mutableListOf() - - for (i in 1..this.metaData.columnCount) { - val fieldName = this.metaData.getColumnName(i) - val fieldType = this.metaData.getColumnTypeName(i) - columns.add(Column(fieldName, fieldType)) - } - - return columns - } -} diff --git a/integrations/sql/src/main/kotlin/com/xebia/functional/xef/sql/SQL.kt b/integrations/sql/src/main/kotlin/com/xebia/functional/xef/sql/SQL.kt deleted file mode 100644 index 6aceaf222..000000000 --- a/integrations/sql/src/main/kotlin/com/xebia/functional/xef/sql/SQL.kt +++ /dev/null @@ -1,160 +0,0 @@ -package com.xebia.functional.xef.sql - -import com.xebia.functional.openai.generated.api.Chat -import com.xebia.functional.openai.generated.model.CreateChatCompletionRequestModel -import com.xebia.functional.xef.Tool -import com.xebia.functional.xef.conversation.AiDsl -import com.xebia.functional.xef.conversation.Conversation -import com.xebia.functional.xef.conversation.Description -import com.xebia.functional.xef.llm.* -import com.xebia.functional.xef.prompt.Prompt -import com.xebia.functional.xef.prompt.PromptBuilder.Companion.system -import com.xebia.functional.xef.prompt.PromptBuilder.Companion.user -import com.xebia.functional.xef.sql.ResultSetOps.toQueryResult -import com.xebia.functional.xef.sql.jdbc.JdbcConfig -import io.github.oshai.kotlinlogging.KotlinLogging -import kotlinx.serialization.Serializable -import kotlinx.serialization.serializer -import org.jetbrains.exposed.sql.Database -import org.jetbrains.exposed.sql.name -import org.jetbrains.exposed.sql.transactions.transaction -import java.time.LocalDate -import java.time.format.DateTimeFormatter - -interface SQL { - companion object { - suspend fun fromJdbcConfig(config: JdbcConfig, block: suspend SQL.() -> A): A = block( - SQLImpl( - config.chatApi, - config.model, - Database.connect(url = config.toJDBCUrl(), user = config.username, password = config.password) - ) - ) - } - - /** - * Generates a SQL query and obtains the table data from a user's prompt. - * - * @param prompt The input prompt. - * @param tableNames A list of table names that may be needed for query generation. - * @param context Additional context information. - * @return An AnswerResponse object containing the response, query details, and result tables. - */ - @AiDsl - suspend fun Conversation.promptQuery(prompt: String, tableNames: List, context: String?): AnswerResponse -} - -class SQLImpl(private val chatApi: Chat, private val model: CreateChatCompletionRequestModel, private val db: Database) : SQL { - private val logger = KotlinLogging.logger {} - - override suspend fun Conversation.promptQuery( - prompt: String, - tableNames: List, - context: String? - ): AnswerResponse { - val queriesAnswer = query(prompt, tableNames, context) - val mainTable = queriesAnswer.mainQuery?.takeIf { it.isNotBlank() }?.let { executeSQL(it) } - val detailedTable = queriesAnswer.detailedQuery?.takeIf { it.isNotBlank() }?.let { executeSQL(it) } - val friendlyResponse = queriesAnswer.replaceFriendlyResponse(mainTable) - - logger.info { "Main query: ${queriesAnswer.mainQuery}" } - logger.info { "Detailed query: ${queriesAnswer.detailedQuery}" } - logger.info { "Friendly response: $friendlyResponse" } - - return AnswerResponse( - input = prompt, - answer = friendlyResponse, - mainQuery = queriesAnswer.mainQuery, - detailedQuery = queriesAnswer.detailedQuery, - mainTable = mainTable, - detailedTable = detailedTable - ) - } - - private fun executeSQL(sql: String): QueryResult = transaction { - runCatching { connection.prepareStatement(sql, false).executeQuery().toQueryResult() }.getOrElse { - logger.info { "Failing executing SQL query: $sql" } - QueryResult.empty() - } - } - - private suspend fun Conversation.query( - input: String, - tableNames: List, - context: String? - ): QueriesAnswer { - val prompt = Prompt(model) { - +system( - """ - As an SQL expert, your main goal is to generate SQL queries, but you must be able to answer any SQL-related question that solves the input. - - Keep into account today's date is ${LocalDate.now().format(DateTimeFormatter.ofPattern("yyyy-MM-dd"))} - The queries have to be compatible with ${db.vendor} in the version ${db.version}. - You must always use aliases to generate the queries. - - The database name is: ${db.name} - We have the tables named: ${tableNames.joinToString(", ")} whose SQL schema are the next: - ${tableDDL(tableNames)} - - "${context?.let { "Analyse the following context to accurate the answer: $it" }}" - - Please check the fields of the tables to be able to generate consistent and valid SQL results. - Make sure that the result includes all the mandatory fields, and analyze if the optional ones are needed. - - In case you have to generate a SQL query to solve the input: - - Select from this list of tables the SQL tables that you need to generate the query. - - Generate a main SQL query that has to satisfy the input of the user if it's possible. - - If the SQL query is successfully generated, provide a friendly sentence summary; otherwise, provide a friendly notice of failure. - - If the query generated returns an amount, the friendly sentence can refer the data with XXX and you have to generate a another query to show the disaggregated data. - - If the friendly sentence refers the data with XXX, add the column name to extract the value to replace it when I run the query. - """.trimIndent() - ) - +user( - """ - |```input - |$input - |``` - """.trimIndent() - ) - } - - return chatApi.prompt( - prompt = prompt, - scope = this, - serializer = Tool.fromKotlin(), - tools = emptyList() - ) - } - -} - - -@Serializable -@Description("SQL queries") -data class QueriesAnswer( - @Description("The main SQL that satisfies the input of the user.") - val mainQuery: String?, - @Description("Friendly sentence that summarize the output.") - val friendlyResponse: String, - @Description("Column name to extract the data to replace the placeholder.") - val columnToReplace: String?, - @Description("SQL to show the disaggregated data.") - val detailedQuery: String? -) { - fun replaceFriendlyResponse(result: QueryResult?): String = - if (friendlyResponse.contains("XXX")) { - val value = columnToReplace?.let { colName -> - result?.let { r -> r.rows.firstOrNull()?.elementAtOrNull(r.index(colName)) } - } ?: "" - friendlyResponse.replace("XXX", value) - } else friendlyResponse -} - -data class AnswerResponse( - val input: String, - val answer: String, - val mainQuery: String?, - val detailedQuery: String?, - val mainTable: QueryResult?, - val detailedTable: QueryResult? -) diff --git a/integrations/sql/src/main/kotlin/com/xebia/functional/xef/sql/jdbc/JdbcConfig.kt b/integrations/sql/src/main/kotlin/com/xebia/functional/xef/sql/jdbc/JdbcConfig.kt deleted file mode 100644 index c740a3dc5..000000000 --- a/integrations/sql/src/main/kotlin/com/xebia/functional/xef/sql/jdbc/JdbcConfig.kt +++ /dev/null @@ -1,17 +0,0 @@ -package com.xebia.functional.xef.sql.jdbc - -import com.xebia.functional.openai.generated.api.Chat -import com.xebia.functional.openai.generated.model.CreateChatCompletionRequestModel - -class JdbcConfig( - val vendor: String, - val host: String, - val username: String, - val password: String, - val port: Int, - val database: String, - val chatApi: Chat, - val model: CreateChatCompletionRequestModel -) { - fun toJDBCUrl(): String = "jdbc:$vendor://$host:$port/$database" -} diff --git a/settings.gradle.kts b/settings.gradle.kts index dbb1da79e..219e615e6 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -43,9 +43,6 @@ project(":xef-pdf").projectDir = file("integrations/pdf") include("xef-postgresql") project(":xef-postgresql").projectDir = file("integrations/postgresql") -include("xef-sql") -project(":xef-sql").projectDir = file("integrations/sql") - include("xef-opentelemetry") project(":xef-opentelemetry").projectDir = file("integrations/opentelemetry")