diff --git a/data-connector/config.json b/data-connector/config.json index 26c2d3f6..c765c590 100644 --- a/data-connector/config.json +++ b/data-connector/config.json @@ -1,6 +1,7 @@ { "privacyMode": 0, "paranoidMode": 0, + "maxResultLength": 20000, "connectors": [ { "type": "postgres", diff --git a/data-connector/package-lock.json b/data-connector/package-lock.json index 5ef63780..f7f69cef 100644 --- a/data-connector/package-lock.json +++ b/data-connector/package-lock.json @@ -14,7 +14,6 @@ "graphql-tag": "^2.12.6", "inferable": "^0.30.48", "mysql2": "^3.11.5", - "pg": "^8.13.1", "sqlite": "^5.1.1", "sqlite3": "^5.1.7", "tsx": "^4.19.2" @@ -22,6 +21,8 @@ "devDependencies": { "@faker-js/faker": "^9.1.0", "@types/node": "^20.11.19", + "@types/pg": "^8.11.10", + "pg": "^8.13.1", "typescript": "^5.3.3" } }, @@ -576,6 +577,80 @@ "undici-types": "~6.19.2" } }, + "node_modules/@types/pg": { + "version": "8.11.10", + "resolved": "https://registry.npmjs.org/@types/pg/-/pg-8.11.10.tgz", + "integrity": "sha512-LczQUW4dbOQzsH2RQ5qoeJ6qJPdrcM/DcMLoqWQkMLMsq83J5lAX3LXjdkWdpscFy67JSOWDnh7Ny/sPFykmkg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*", + "pg-protocol": "*", + "pg-types": "^4.0.1" + } + }, + "node_modules/@types/pg/node_modules/pg-types": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-4.0.2.tgz", + "integrity": "sha512-cRL3JpS3lKMGsKaWndugWQoLOCoP+Cic8oseVcbr0qhPzYD5DWXK+RZ9LY9wxRf7RQia4SCwQlXk0q6FCPrVng==", + "dev": true, + "license": "MIT", + "dependencies": { + "pg-int8": "1.0.1", + "pg-numeric": "1.0.2", + "postgres-array": "~3.0.1", + "postgres-bytea": "~3.0.0", + "postgres-date": "~2.1.0", + "postgres-interval": "^3.0.0", + "postgres-range": "^1.1.1" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/@types/pg/node_modules/postgres-array": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-3.0.2.tgz", + "integrity": "sha512-6faShkdFugNQCLwucjPcY5ARoW1SlbnrZjmGl0IrrqewpvxvhSLHimCVzqeuULCbG0fQv7Dtk1yDbG3xv7Veog==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + } + }, + "node_modules/@types/pg/node_modules/postgres-bytea": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-3.0.0.tgz", + "integrity": "sha512-CNd4jim9RFPkObHSjVHlVrxoVQXz7quwNFpz7RY1okNNme49+sVyiTvTRobiLV548Hx/hb1BG+iE7h9493WzFw==", + "dev": true, + "license": "MIT", + "dependencies": { + "obuf": "~1.1.2" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/@types/pg/node_modules/postgres-date": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-2.1.0.tgz", + "integrity": "sha512-K7Juri8gtgXVcDfZttFKVmhglp7epKb1K4pgrkLxehjqkrgPhfG6OO8LHLkfaqkbpjNRnra018XwAr1yQFWGcA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + } + }, + "node_modules/@types/pg/node_modules/postgres-interval": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-3.0.0.tgz", + "integrity": "sha512-BSNDnbyZCXSxgA+1f5UU2GmwhoI0aU5yMxRGO8CdFEcY2BQF9xm/7MqKnYoM1nJDk8nONNWDk9WeSmePFhQdlw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + } + }, "node_modules/abbrev": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", @@ -1937,6 +2012,13 @@ "node": "^12.13.0 || ^14.15.0 || >=16.0.0" } }, + "node_modules/obuf": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/obuf/-/obuf-1.1.2.tgz", + "integrity": "sha512-PX1wu0AmAdPqOL1mWhqmlOd8kOIZQwGZw6rh7uby9fTc5lhaOWFLX3I6R1hrF9k3zUY40e6igsLGkDXK92LJNg==", + "dev": true, + "license": "MIT" + }, "node_modules/on-exit-leak-free": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/on-exit-leak-free/-/on-exit-leak-free-2.1.2.tgz", @@ -1985,6 +2067,7 @@ "version": "8.13.1", "resolved": "https://registry.npmjs.org/pg/-/pg-8.13.1.tgz", "integrity": "sha512-OUir1A0rPNZlX//c7ksiu7crsGZTKSOXJPgtNiHGIlC9H0lO+NC6ZDYksSgBYY/thSWhnSRBv8w1lieNNGATNQ==", + "dev": true, "license": "MIT", "dependencies": { "pg-connection-string": "^2.7.0", @@ -2012,6 +2095,7 @@ "version": "1.1.1", "resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.1.1.tgz", "integrity": "sha512-xWPagP/4B6BgFO+EKz3JONXv3YDgvkbVrGw2mTo3D6tVDQRh1e7cqVGvyR3BE+eQgAvx1XhW/iEASj4/jCWl3Q==", + "dev": true, "license": "MIT", "optional": true }, @@ -2019,21 +2103,34 @@ "version": "2.7.0", "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.7.0.tgz", "integrity": "sha512-PI2W9mv53rXJQEOb8xNR8lH7Hr+EKa6oJa38zsK0S/ky2er16ios1wLKhZyxzD7jUReiWokc9WK5nxSnC7W1TA==", + "dev": true, "license": "MIT" }, "node_modules/pg-int8": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==", + "dev": true, "license": "ISC", "engines": { "node": ">=4.0.0" } }, + "node_modules/pg-numeric": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/pg-numeric/-/pg-numeric-1.0.2.tgz", + "integrity": "sha512-BM/Thnrw5jm2kKLE5uJkXqqExRUY/toLHda65XgFTBTFYZyopbKjBe29Ii3RbkvlsMoFwD+tHeGaCjjv0gHlyw==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=4" + } + }, "node_modules/pg-pool": { "version": "3.7.0", "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.7.0.tgz", "integrity": "sha512-ZOBQForurqh4zZWjrgSwwAtzJ7QiRX0ovFkZr2klsen3Nm0aoh33Ls0fzfv3imeH/nw/O27cjdz5kzYJfeGp/g==", + "dev": true, "license": "MIT", "peerDependencies": { "pg": ">=8.0" @@ -2043,12 +2140,14 @@ "version": "1.7.0", "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.7.0.tgz", "integrity": "sha512-hTK/mE36i8fDDhgDFjy6xNOG+LCorxLG3WO17tku+ij6sVHXh1jQUJ8hYAnRhNla4QVD2H8er/FOjc/+EgC6yQ==", + "dev": true, "license": "MIT" }, "node_modules/pg-types": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz", "integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==", + "dev": true, "license": "MIT", "dependencies": { "pg-int8": "1.0.1", @@ -2065,6 +2164,7 @@ "version": "1.0.5", "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz", "integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==", + "dev": true, "license": "MIT", "dependencies": { "split2": "^4.1.0" @@ -2117,6 +2217,7 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz", "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==", + "dev": true, "license": "MIT", "engines": { "node": ">=4" @@ -2126,6 +2227,7 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.0.tgz", "integrity": "sha512-xy3pmLuQqRBZBXDULy7KbaitYqLcmxigw14Q5sj8QBVLqEwXfeybIKVWiqAXTlcvdvb0+xkOtDbfQMOf4lST1w==", + "dev": true, "license": "MIT", "engines": { "node": ">=0.10.0" @@ -2135,6 +2237,7 @@ "version": "1.0.7", "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz", "integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==", + "dev": true, "license": "MIT", "engines": { "node": ">=0.10.0" @@ -2144,6 +2247,7 @@ "version": "1.2.0", "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz", "integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==", + "dev": true, "license": "MIT", "dependencies": { "xtend": "^4.0.0" @@ -2152,6 +2256,13 @@ "node": ">=0.10.0" } }, + "node_modules/postgres-range": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/postgres-range/-/postgres-range-1.1.4.tgz", + "integrity": "sha512-i/hbxIE9803Alj/6ytL7UHQxRvZkI9O4Sy+J3HGc4F4oo/2eQAjTSNJ0bfxyse3bH0nuVesCk+3IRLaMtG3H6w==", + "dev": true, + "license": "MIT" + }, "node_modules/prebuild-install": { "version": "7.1.2", "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.2.tgz", @@ -2851,6 +2962,7 @@ "version": "4.0.2", "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", + "dev": true, "license": "MIT", "engines": { "node": ">=0.4" diff --git a/data-connector/package.json b/data-connector/package.json index 017fd4cf..ee428d32 100644 --- a/data-connector/package.json +++ b/data-connector/package.json @@ -16,7 +16,6 @@ "graphql-tag": "^2.12.6", "inferable": "^0.30.48", "mysql2": "^3.11.5", - "pg": "^8.13.1", "sqlite": "^5.1.1", "sqlite3": "^5.1.7", "tsx": "^4.19.2" @@ -24,6 +23,8 @@ "devDependencies": { "@faker-js/faker": "^9.1.0", "@types/node": "^20.11.19", + "@types/pg": "^8.11.10", + "pg": "^8.13.1", "typescript": "^5.3.3" } } diff --git a/data-connector/src/graphql/graphql.ts b/data-connector/src/graphql/graphql.ts index e9de219f..ebedcfe1 100644 --- a/data-connector/src/graphql/graphql.ts +++ b/data-connector/src/graphql/graphql.ts @@ -25,6 +25,7 @@ export class GraphQLClient implements DataConnector { name?: string; schemaUrl: string; endpoint: string; + maxResultLength: number; defaultHeaders?: Record; privacyMode: boolean; paranoidMode: boolean; @@ -176,6 +177,18 @@ To understand the input and output types for this operation, use the searchGraph }; } + if (JSON.stringify(data).length > this.params.maxResultLength) { + return { + message: + "This query returned too much data. Data was returned to the user directly.", + blob: blob({ + name: "Results", + type: "application/json", + data: data, + }), + }; + } + return data; }; diff --git a/data-connector/src/index.ts b/data-connector/src/index.ts index bd3df635..b52521c7 100644 --- a/data-connector/src/index.ts +++ b/data-connector/src/index.ts @@ -51,9 +51,14 @@ const parseConfig = (connector: any) => { continue; } + if (!!connector.maxResultLength && isNaN(Number(connector.maxResultLength))) { + throw new Error("maxResultLength must be a number"); + } + if (connector.type === "postgres") { const postgresClient = new PostgresClient({ ...connector, + maxResultLength: Number(config.maxResultLength), paranoidMode: config.paranoidMode === 1, privacyMode: config.privacyMode === 1, }); @@ -63,6 +68,7 @@ const parseConfig = (connector: any) => { } else if (connector.type === "open-api") { const openAPIClient = new OpenAPIClient({ ...connector, + maxResultLength: Number(config.maxResultLength), paranoidMode: config.paranoidMode === 1, privacyMode: config.privacyMode === 1, }); @@ -72,6 +78,7 @@ const parseConfig = (connector: any) => { } else if (connector.type === "graphql") { const graphQLClient = new GraphQLClient({ ...connector, + maxResultLength: Number(config.maxResultLength), paranoidMode: config.paranoidMode === 1, privacyMode: config.privacyMode === 1, }); @@ -81,6 +88,7 @@ const parseConfig = (connector: any) => { } else if (connector.type === "mysql") { const mysqlClient = new MySQLClient({ ...connector, + maxResultLength: Number(config.maxResultLength), paranoidMode: config.paranoidMode === 1, privacyMode: config.privacyMode === 1, }); @@ -90,6 +98,7 @@ const parseConfig = (connector: any) => { } else if (connector.type === "sqlite") { const sqliteClient = new SQLiteClient({ ...connector, + maxResultLength: Number(config.maxResultLength), paranoidMode: config.paranoidMode === 1, privacyMode: config.privacyMode === 1, }); diff --git a/data-connector/src/mysql/mysql.ts b/data-connector/src/mysql/mysql.ts index fa835c24..e8309f5c 100644 --- a/data-connector/src/mysql/mysql.ts +++ b/data-connector/src/mysql/mysql.ts @@ -14,6 +14,7 @@ export class MySQLClient implements DataConnector { name?: string; schema: string; connectionString: string; + maxResultLength: number; privacyMode: boolean; paranoidMode: boolean; }, @@ -131,6 +132,18 @@ export class MySQLClient implements DataConnector { }; } + if (JSON.stringify(rows).length > this.params.maxResultLength) { + return { + message: + "This query returned too much data. Data was returned to the user directly.", + blob: blob({ + name: "Results", + type: "application/json", + data: rows, + }), + }; + } + return rows; }; diff --git a/data-connector/src/open-api/open-api.ts b/data-connector/src/open-api/open-api.ts index 9d7936d1..b2c6fd64 100644 --- a/data-connector/src/open-api/open-api.ts +++ b/data-connector/src/open-api/open-api.ts @@ -18,6 +18,7 @@ export class OpenAPIClient implements DataConnector { name?: string; specUrl: string; endpoint?: string; + maxResultLength: number; defaultHeaders?: Record; privacyMode: boolean; paranoidMode: boolean; @@ -232,6 +233,18 @@ export class OpenAPIClient implements DataConnector { }; } + if (JSON.stringify(parsed).length > this.params.maxResultLength) { + return { + message: + "This query returned too much data. Data was returned to the user directly.", + blob: blob({ + name: "Results", + type: "application/json", + data: parsed, + }), + }; + } + return parsed; }; diff --git a/data-connector/src/postgres/postgres.ts b/data-connector/src/postgres/postgres.ts index 5c988b39..a21cd6e9 100644 --- a/data-connector/src/postgres/postgres.ts +++ b/data-connector/src/postgres/postgres.ts @@ -14,6 +14,7 @@ export class PostgresClient implements DataConnector { name?: string; schema: string; connectionString: string; + maxResultLength: number; privacyMode: boolean; paranoidMode: boolean; }, @@ -138,6 +139,18 @@ export class PostgresClient implements DataConnector { }; } + if (JSON.stringify(res.rows).length > this.params.maxResultLength) { + return { + message: + "This query returned too much data. Data was returned to the user directly.", + blob: blob({ + name: "Results", + type: "application/json", + data: res.rows, + }), + }; + } + return res.rows; }; diff --git a/data-connector/src/sqlite/sqlite.ts b/data-connector/src/sqlite/sqlite.ts index ccae3e38..11366179 100644 --- a/data-connector/src/sqlite/sqlite.ts +++ b/data-connector/src/sqlite/sqlite.ts @@ -14,6 +14,7 @@ export class SQLiteClient implements DataConnector { private params: { name?: string; filePath: string; + maxResultLength: number; privacyMode: boolean; paranoidMode: boolean; }, @@ -129,6 +130,18 @@ export class SQLiteClient implements DataConnector { }; } + if (JSON.stringify(rows).length > this.params.maxResultLength) { + return { + message: + "This query returned too much data. Data was returned to the user directly.", + blob: blob({ + name: "Results", + type: "application/json", + data: rows, + }), + }; + } + return rows; };