diff --git a/data-connector/package-lock.json b/data-connector/package-lock.json index 525be1d5..591519b7 100644 --- a/data-connector/package-lock.json +++ b/data-connector/package-lock.json @@ -13,6 +13,7 @@ "graphql": "^16.9.0", "graphql-tag": "^2.12.6", "inferable": "^0.30.58", + "js-yaml": "^4.1.0", "mysql2": "^3.11.5", "sqlite": "^5.1.1", "sqlite3": "^5.1.7", @@ -20,8 +21,10 @@ }, "devDependencies": { "@faker-js/faker": "^9.1.0", + "@types/js-yaml": "^4.0.9", "@types/node": "^20.11.19", "@types/pg": "^8.11.10", + "openapi-types": "^12.1.3", "pg": "^8.13.1", "typescript": "^5.3.3" } @@ -555,6 +558,13 @@ "@types/ms": "*" } }, + "node_modules/@types/js-yaml": { + "version": "4.0.9", + "resolved": "https://registry.npmjs.org/@types/js-yaml/-/js-yaml-4.0.9.tgz", + "integrity": "sha512-k4MGaQl5TGo/iipqb2UDG2UwjXziSWkh0uysQelTlJpX1qGlpUZYm8PnO4DxG1qBomtJUdYJ6qR6xdIah10JLg==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/json-schema": { "version": "7.0.15", "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", @@ -769,6 +779,12 @@ "node": "^12.13.0 || ^14.15.0 || >=16.0.0" } }, + "node_modules/argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", + "license": "Python-2.0" + }, "node_modules/atomic-sleep": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/atomic-sleep/-/atomic-sleep-1.0.0.tgz", @@ -1625,6 +1641,18 @@ "license": "ISC", "optional": true }, + "node_modules/js-yaml": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", + "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", + "license": "MIT", + "dependencies": { + "argparse": "^2.0.1" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, "node_modules/jsbn": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-1.1.0.tgz", @@ -2037,6 +2065,13 @@ "wrappy": "1" } }, + "node_modules/openapi-types": { + "version": "12.1.3", + "resolved": "https://registry.npmjs.org/openapi-types/-/openapi-types-12.1.3.tgz", + "integrity": "sha512-N4YtSYJqghVu4iek2ZUvcN/0aqH1kRDuNqzcycDxhOUpg7GdvLa2F3DgS6yBNhInhv2r/6I0Flkn7CqL8+nIcw==", + "dev": true, + "license": "MIT" + }, "node_modules/p-map": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/p-map/-/p-map-4.0.0.tgz", diff --git a/data-connector/package.json b/data-connector/package.json index 924e44ca..c5a9552e 100644 --- a/data-connector/package.json +++ b/data-connector/package.json @@ -15,6 +15,7 @@ "graphql": "^16.9.0", "graphql-tag": "^2.12.6", "inferable": "^0.30.58", + "js-yaml": "^4.1.0", "mysql2": "^3.11.5", "sqlite": "^5.1.1", "sqlite3": "^5.1.7", @@ -22,8 +23,10 @@ }, "devDependencies": { "@faker-js/faker": "^9.1.0", + "@types/js-yaml": "^4.0.9", "@types/node": "^20.11.19", "@types/pg": "^8.11.10", + "openapi-types": "^12.1.3", "pg": "^8.13.1", "typescript": "^5.3.3" } diff --git a/data-connector/src/open-api/open-api.ts b/data-connector/src/open-api/open-api.ts index b2c6fd64..721ccb75 100644 --- a/data-connector/src/open-api/open-api.ts +++ b/data-connector/src/open-api/open-api.ts @@ -1,10 +1,11 @@ import { approvalRequest, blob, ContextInput, Inferable } from "inferable"; import { z } from "zod"; import type { DataConnector } from "../types"; -import { OpenAPIV3 } from "openapi-types"; +import { OpenAPI, OpenAPIV3 } from "openapi-types"; import crypto from "crypto"; import { FunctionRegistrationInput } from "inferable/bin/types"; import assert from "assert"; +import yaml from "js-yaml"; export class OpenAPIClient implements DataConnector { private spec: OpenAPIV3.Document | null = null; @@ -27,8 +28,8 @@ export class OpenAPIClient implements DataConnector { public initialize = async () => { try { - const response = await fetch(this.params.specUrl); - this.spec = (await response.json()) as OpenAPIV3.Document; + // Handle Yaml or JSON + this.spec = await this.fetchOpenAPISchema(this.params.specUrl); console.log( `OpenAPI spec loaded successfully from ${this.params.specUrl}`, ); @@ -120,19 +121,15 @@ export class OpenAPIClient implements DataConnector { `${method.toUpperCase()} ${path}`; return { - name: operation.operationId, + name: this.camelCase(operation.operationId), description: `${summary}. Ask the user to provide values for any required parameters.`, - func: this.executeRequest, + func: this.executeRequest({ + path, + method, + parametersByLocation, + }), schema: { input: z.object({ - path: path.includes(":") - ? z - .string() - .describe( - `Must be ${path} with values substituted for any path parameters.`, - ) - : z.literal(path), - method: z.literal(method.toUpperCase()), parameters: hasParameters ? z .record(z.any()) @@ -156,10 +153,16 @@ export class OpenAPIClient implements DataConnector { }; }; - executeRequest = async ( + executeRequest = (endpoint: { + path: string; + method: string; + parametersByLocation: { + path: string[]; + query: string[]; + header: string[]; + } + }) => async ( input: { - path: string; - method: string; parameters?: Record; body?: any; }, @@ -183,16 +186,34 @@ export class OpenAPIClient implements DataConnector { this.spec.servers?.[0]?.url || "" ).toString(); - let finalPath = input.path; + + let finalPath = endpoint.path; if (input.parameters) { + // Replace path parameters - Object.entries(input.parameters).forEach(([key, value]) => { - finalPath = finalPath.replace( - `{${key}}`, - encodeURIComponent(String(value)), - ); - }); + let pathParameters: string[] = []; + finalPath = Object + .entries(input.parameters) + .filter(([key]) => endpoint.parametersByLocation.path.includes(key)) + .reduce((path, [key, value]) => { + if (path.includes(`{${key}}`)) { + pathParameters.push(key); + } + + return path.replace(`{${key}}`, encodeURIComponent(String(value))); + }, finalPath); + + // Add any query parameters + finalPath += '?' + Object + .entries(input.parameters) + .filter(([key]) => endpoint.parametersByLocation.query.includes(key)) + .filter(([key]) => !pathParameters.includes(key)) + .reduce( + (params, [key, value]) => { + params.set(key, String(value)); + return params; + }, new URLSearchParams()).toString(); } url += finalPath; @@ -203,8 +224,17 @@ export class OpenAPIClient implements DataConnector { ...this.params.defaultHeaders, }; + if (input.parameters) { + // Add any additional headers + Object.entries(input.parameters) + .filter(([key]) => endpoint.parametersByLocation.header.includes(key)) + .forEach(([key, value]) => { + headers[key] = String(value); + }); + } + const response = await fetch(url, { - method: input.method, + method: endpoint.method, headers, body: input.body ? JSON.stringify(input.body) : undefined, }); @@ -267,4 +297,42 @@ export class OpenAPIClient implements DataConnector { return service; }; + + private camelCase = (operationId: string) => { + return operationId + .split("_") + .map((part) => part.charAt(0).toUpperCase() + part.slice(1)) + .join(""); + } + + private fetchOpenAPISchema = async ( + url: string, + ): Promise => { + try { + const response = await fetch(url); + + if (!response.ok) { + throw new Error(`Failed to fetch schema: ${response.statusText}`); + } + + const contentType = response.headers.get('content-type') || ''; + const isJson = contentType.includes('application/json') || url.endsWith('.json'); + const isYaml = contentType.includes('application/x-yaml') || url.endsWith('.yaml') || url.endsWith('.yml'); + + const schemaText = await response.text(); + + if (isJson) { + return JSON.parse(schemaText) as OpenAPIV3.Document; + } else if (isYaml) { + return yaml.load(schemaText) as OpenAPIV3.Document; + } else { + throw new Error(`Unsupported format or mismatch between requested and detected format.`); + } + } catch (error) { + console.error('Error downloading OpenAPI schema:', error); + throw error; + } + } } + +