feat: SQLite data connector

data-connector/.env.example

@@ -28,3 +28,8 @@ POSTGRES_SCHEMA=public
 #
 # MYSQL_URL=mysql://root:mysql@localhost:3306/your_database
 # MYSQL_SCHEMA=your_database
+
+#
+# SQLite config example.
+#
+# SQLITE_PATH=/path/to/your/database.sqlite

data-connector/README.md

@@ -17,7 +17,14 @@ Inferable Data Connector is a bridge between your data systems and Inferable. Co
 - [x] [OpenAPI](./src/open-api/open-api.ts)
 - [x] [GraphQL](./src/graphql/graphql.ts)
 - [x] [MySQL](./src/mysql/mysql.ts)
+<<<<<<< HEAD
+- [x] [SQLite](./src/sqlite/sqlite.ts)
+- [ ] [MongoDB](./src/mongodb/mongodb.ts)
+- [ ] [Big Query](./src/big-query/big-query.ts)
+- [ ] [Google Sheets](./src/google-sheets/google-sheets.ts)
+=======
 - [ ] [SQLite](./src/sqlite/sqlite.ts)
+>>>>>>> origin/main
 
 ## Quick Start
 
@@ -131,6 +138,16 @@ Each connector is defined in the `config.connectors` array.
 
 </details>
 
+<<<<<<< HEAD
+<details>
+<summary>SQLite Connector Configuration</summary>
+
+- `config.connectors[].filePath`: The path to your SQLite database file. (e.g. `/path/to/your/database.sqlite`)
+
+</details>
+
+=======
+>>>>>>> origin/main
 ### config.privacyMode
 
 When enabled (`config.privacyMode=1`), raw data is never sent to the model. Instead:

data-connector/config.json

@@ -31,6 +31,11 @@
       "name": "myMysql",
       "connectionString": "process.env.MYSQL_URL",
       "schema": "process.env.MYSQL_SCHEMA"
+    },
+    {
+      "type": "sqlite",
+      "name": "mySqlite",
+      "filePath": "process.env.SQLITE_PATH"
     }
   ]
 }

data-connector/example_data/seed-sqlite.sh

@@ -0,0 +1,88 @@
+#!/bin/bash
+
+# Database name
+DB_NAME="sample.db"
+
+# Remove database if it exists
+rm -f $DB_NAME
+
+# Create database and tables using heredoc
+sqlite3 $DB_NAME << 'END_SQL'
+-- Create tables
+CREATE TABLE users (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    name TEXT NOT NULL,
+    email TEXT UNIQUE,
+    created_at DATETIME DEFAULT CURRENT_TIMESTAMP
+);
+
+CREATE TABLE posts (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    user_id INTEGER,
+    title TEXT NOT NULL,
+    content TEXT,
+    created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
+    FOREIGN KEY (user_id) REFERENCES users(id)
+);
+
+CREATE TABLE comments (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    post_id INTEGER,
+    user_id INTEGER,
+    content TEXT NOT NULL,
+    created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
+    FOREIGN KEY (post_id) REFERENCES posts(id),
+    FOREIGN KEY (user_id) REFERENCES users(id)
+);
+
+-- Insert sample data
+INSERT INTO users (name, email) VALUES
+    ('John Doe', '[email protected]'),
+    ('Jane Smith', '[email protected]'),
+    ('Bob Wilson', '[email protected]');
+
+INSERT INTO posts (user_id, title, content) VALUES
+    (1, 'First Post', 'This is my first post content'),
+    (1, 'Second Post', 'Another interesting post'),
+    (2, 'Hello World', 'Jane''s first blog post'),
+    (3, 'Tech Review', 'Review of the latest gadget');
+
+INSERT INTO comments (post_id, user_id, content) VALUES
+    (1, 2, 'Great first post!'),
+    (1, 3, 'Welcome to the blog'),
+    (2, 2, 'Interesting perspective'),
+    (3, 1, 'Thanks for sharing'),
+    (4, 2, 'Very informative review');
+
+-- Create some indexes
+CREATE INDEX idx_posts_user_id ON posts(user_id);
+CREATE INDEX idx_comments_post_id ON comments(post_id);
+CREATE INDEX idx_comments_user_id ON comments(user_id);
+
+-- Show some sample queries
+SELECT 'Sample Query 1: Posts with author names and comment counts';
+SELECT
+    p.title,
+    u.name as author,
+    COUNT(c.id) as comment_count
+FROM posts p
+JOIN users u ON p.user_id = u.id
+LEFT JOIN comments c ON p.id = c.post_id
+GROUP BY p.id
+ORDER BY p.id;
+
+SELECT '
+Sample Query 2: Recent comments with post titles and commenter names';
+SELECT
+    c.content as comment,
+    p.title as post_title,
+    u.name as commenter
+FROM comments c
+JOIN posts p ON c.post_id = p.id
+JOIN users u ON c.user_id = u.id
+ORDER BY c.created_at DESC
+LIMIT 5;
+END_SQL
+
+echo "Database $DB_NAME has been created with sample data!"
+echo "You can connect to it using: sqlite3 $DB_NAME"

data-connector/src/index.ts

@@ -6,6 +6,7 @@ import { RegisteredService } from "inferable/bin/types";
 import { OpenAPIClient } from "./open-api/open-api";
 import { GraphQLClient } from "./graphql/graphql";
 import { MySQLClient } from "./mysql/mysql";
+import { SQLiteClient } from "./sqlite/sqlite";
 
 const parseConfig = (connector: any) => {
   for (const [key, value] of Object.entries(connector)) {
@@ -73,6 +74,15 @@ const parseConfig = (connector: any) => {
       await mysqlClient.initialize();
       const service = mysqlClient.createService(client);
       services.push(service);
+    } else if (connector.type === "sqlite") {
+      const sqliteClient = new SQLiteClient({
+        ...connector,
+        paranoidMode: config.paranoidMode === 1,
+        privacyMode: config.privacyMode === 1,
+      });
+      await sqliteClient.initialize();
+      const service = sqliteClient.createService(client);
+      services.push(service);
     } else {
       throw new Error(`Unknown connector type: ${connector.type}`);
     }

data-connector/src/postgres/postgres.ts

@@ -3,11 +3,11 @@ import { approvalRequest, blob, ContextInput, Inferable } from "inferable";
 import pg from "pg";
 import { z } from "zod";
 import crypto from "crypto";
-import type { DataConnector } from "./types";
+import type { DataConnector } from "../types";
 
 export class PostgresClient implements DataConnector {
   private client: pg.Client | null = null;
-  private initialized: Promise<void>;
+  private initialized = false;
 
   constructor(
     private params: {
@@ -34,6 +34,7 @@ export class PostgresClient implements DataConnector {
 
       process.removeListener("SIGTERM", this.handleSigterm);
       process.on("SIGTERM", this.handleSigterm);
+      this.initialized = true;
     } catch (error) {
       console.error("Failed to initialize database connection:", error);
       throw error;
@@ -70,8 +71,8 @@ export class PostgresClient implements DataConnector {
     return res.rows;
   };
 
-  getContext = async () => {
-    await this.initialized;
+  getPostgresContext = async () => {
+    if (!this.initialized) throw new Error("Database not initialized");
     const client = await this.getClient();
     const tables = await this.getAllTables();
 
@@ -108,7 +109,10 @@ export class PostgresClient implements DataConnector {
     return context;
   };
 
-  executeQuery = async (input: { query: string }, ctx: ContextInput) => {
+  executePostgresQuery = async (
+    input: { query: string },
+    ctx: ContextInput,
+  ) => {
     if (this.params.paranoidMode) {
       if (!ctx.approved) {
         console.log("Query requires approval");
@@ -118,7 +122,7 @@ export class PostgresClient implements DataConnector {
       }
     }
 
-    await this.initialized;
+    if (!this.initialized) throw new Error("Database not initialized");
     const client = await this.getClient();
     const res = await client.query(input.query);
 
@@ -152,14 +156,14 @@ export class PostgresClient implements DataConnector {
     });
 
     service.register({
-      name: "getContext",
-      func: this.getContext,
+      name: "getPostgresContext",
+      func: this.getPostgresContext,
       description: "Gets the schema of the database.",
     });
 
     service.register({
-      name: "executeQuery",
-      func: this.executeQuery,
+      name: "executePostgresQuery",
+      func: this.executePostgresQuery,
       description:
         "Executes a raw SQL query. If this fails, you need to getContext to learn the schema first.",
       schema: {