feat: Return large results as Blob

data-connector/config.json

@@ -1,6 +1,7 @@
 {
   "privacyMode": 0,
   "paranoidMode": 0,
+  "maxResultLength": 20000,
   "connectors": [
     {
       "type": "postgres",

data-connector/package.json

@@ -16,14 +16,15 @@
     "graphql-tag": "^2.12.6",
     "inferable": "^0.30.48",
     "mysql2": "^3.11.5",
-    "pg": "^8.13.1",
     "sqlite": "^5.1.1",
     "sqlite3": "^5.1.7",
     "tsx": "^4.19.2"
   },
   "devDependencies": {
     "@faker-js/faker": "^9.1.0",
     "@types/node": "^20.11.19",
+    "@types/pg": "^8.11.10",
+    "pg": "^8.13.1",
     "typescript": "^5.3.3"
   }
 }

data-connector/src/graphql/graphql.ts

@@ -25,6 +25,7 @@ export class GraphQLClient implements DataConnector {
       name?: string;
       schemaUrl: string;
       endpoint: string;
+      maxResultLength: number;
       defaultHeaders?: Record<string, string>;
       privacyMode: boolean;
       paranoidMode: boolean;
@@ -176,6 +177,18 @@ To understand the input and output types for this operation, use the searchGraph
       };
     }
 
+    if (JSON.stringify(data).length > this.params.maxResultLength) {
+      return {
+        message:
+          "This query returned too much data. Data was returned to the user directly.",
+        blob: blob({
+          name: "Results",
+          type: "application/json",
+          data: data,
+        }),
+      };
+    }
+
     return data;
   };
 

data-connector/src/index.ts

@@ -51,9 +51,14 @@ const parseConfig = (connector: any) => {
       continue;
     }
 
+    if (!!connector.maxResultLength && isNaN(Number(connector.maxResultLength))) {
+      throw new Error("maxResultLength must be a number");
+    }
+
     if (connector.type === "postgres") {
       const postgresClient = new PostgresClient({
         ...connector,
+        maxResultLength: Number(config.maxResultLength),
         paranoidMode: config.paranoidMode === 1,
         privacyMode: config.privacyMode === 1,
       });
@@ -63,6 +68,7 @@ const parseConfig = (connector: any) => {
     } else if (connector.type === "open-api") {
       const openAPIClient = new OpenAPIClient({
         ...connector,
+        maxResultLength: Number(config.maxResultLength),
         paranoidMode: config.paranoidMode === 1,
         privacyMode: config.privacyMode === 1,
       });
@@ -72,6 +78,7 @@ const parseConfig = (connector: any) => {
     } else if (connector.type === "graphql") {
       const graphQLClient = new GraphQLClient({
         ...connector,
+        maxResultLength: Number(config.maxResultLength),
         paranoidMode: config.paranoidMode === 1,
         privacyMode: config.privacyMode === 1,
       });
@@ -81,6 +88,7 @@ const parseConfig = (connector: any) => {
     } else if (connector.type === "mysql") {
       const mysqlClient = new MySQLClient({
         ...connector,
+        maxResultLength: Number(config.maxResultLength),
         paranoidMode: config.paranoidMode === 1,
         privacyMode: config.privacyMode === 1,
       });
@@ -90,6 +98,7 @@ const parseConfig = (connector: any) => {
     } else if (connector.type === "sqlite") {
       const sqliteClient = new SQLiteClient({
         ...connector,
+        maxResultLength: Number(config.maxResultLength),
         paranoidMode: config.paranoidMode === 1,
         privacyMode: config.privacyMode === 1,
       });

data-connector/src/mysql/mysql.ts

@@ -14,6 +14,7 @@ export class MySQLClient implements DataConnector {
       name?: string;
       schema: string;
       connectionString: string;
+      maxResultLength: number;
       privacyMode: boolean;
       paranoidMode: boolean;
     },
@@ -131,6 +132,18 @@ export class MySQLClient implements DataConnector {
       };
     }
 
+    if (JSON.stringify(rows).length > this.params.maxResultLength) {
+      return {
+        message:
+          "This query returned too much data. Data was returned to the user directly.",
+        blob: blob({
+          name: "Results",
+          type: "application/json",
+          data: rows,
+        }),
+      };
+    }
+
     return rows;
   };
 

data-connector/src/open-api/open-api.ts

@@ -18,6 +18,7 @@ export class OpenAPIClient implements DataConnector {
       name?: string;
       specUrl: string;
       endpoint?: string;
+      maxResultLength: number;
       defaultHeaders?: Record<string, string>;
       privacyMode: boolean;
       paranoidMode: boolean;
@@ -232,6 +233,18 @@ export class OpenAPIClient implements DataConnector {
       };
     }
 
+    if (JSON.stringify(parsed).length > this.params.maxResultLength) {
+      return {
+        message:
+          "This query returned too much data. Data was returned to the user directly.",
+        blob: blob({
+          name: "Results",
+          type: "application/json",
+          data: parsed,
+        }),
+      };
+    }
+
     return parsed;
   };