From d1e6f9241129afaf5728b6c3f72755c456cc470a Mon Sep 17 00:00:00 2001 From: "Tyler B. Thrailkill" Date: Fri, 8 Mar 2024 16:10:37 -0700 Subject: [PATCH 1/3] Optimize the runOpenApi plugin allow passing in an already dereferenced OpenAPI Document for plugins that have large openapi specs like github. This takes the runtime for Github from over 10s to ~1s on an M1 Max. Adjust the github plugin to use this new capability by dereferencing on plugin initialization, passing the derefed document to the runOpenApi function. --- .../node-service/src/plugins/github/index.ts | 6 ++-- .../node-service/src/plugins/openApi/index.ts | 30 ++++++++++++------- 2 files changed, 24 insertions(+), 12 deletions(-) diff --git a/server/node-service/src/plugins/github/index.ts b/server/node-service/src/plugins/github/index.ts index 5df537677..3f031b65e 100644 --- a/server/node-service/src/plugins/github/index.ts +++ b/server/node-service/src/plugins/github/index.ts @@ -5,6 +5,7 @@ import { OpenAPIV3, OpenAPI } from "openapi-types"; import { ConfigToType, DataSourcePlugin } from "lowcoder-sdk/dataSource"; import { runOpenApi } from "../openApi"; import { parseOpenApi, ParseOpenApiOptions } from "../openApi/parse"; +import SwaggerParser from "@apidevtools/swagger-parser"; const spec = readYaml(path.join(__dirname, "./github.spec.yaml")); @@ -34,6 +35,7 @@ const parseOptions: ParseOpenApiOptions = { return _.upperFirst(operation.operationId || ""); }, }; +const deRefedSpec = SwaggerParser.dereference(spec); type DataSourceConfigType = ConfigToType; @@ -55,13 +57,13 @@ const gitHubPlugin: DataSourcePlugin = { actions, }; }, - run: function (actionData, dataSourceConfig): Promise { + run: async function (actionData, dataSourceConfig, ctx): Promise { const runApiDsConfig = { url: "", serverURL: "https://api.github.com", dynamicParamsConfig: dataSourceConfig, }; - return runOpenApi(actionData, runApiDsConfig, spec as OpenAPIV3.Document); + return runOpenApi(actionData, runApiDsConfig, spec as OpenAPIV3.Document, undefined, await deRefedSpec); }, }; diff --git a/server/node-service/src/plugins/openApi/index.ts b/server/node-service/src/plugins/openApi/index.ts index 27f9ed950..eaec12aac 100644 --- a/server/node-service/src/plugins/openApi/index.ts +++ b/server/node-service/src/plugins/openApi/index.ts @@ -54,18 +54,28 @@ export async function runOpenApi( actionData: ActionDataType, dataSourceConfig: DataSourceDataType, spec: OpenAPI.Document | MultiOpenApiSpecItem[], - defaultHeaders?: Record + defaultHeaders?: Record, + openApiSpecDereferenced?: OpenAPI.Document, ) { const specList = Array.isArray(spec) ? spec : [{ spec, id: "" }]; - const definitions = await Promise.all( - specList.map(async ({ id, spec }) => { - const deRefedSpec = await SwaggerParser.dereference(spec); - return { - def: deRefedSpec, - id, - }; - }) - ); + let definitions; + + if (!openApiSpecDereferenced) { + definitions = await Promise.all( + specList.map(async ({id, spec}) => { + const deRefedSpec = await SwaggerParser.dereference(spec); + return { + def: deRefedSpec, + id, + }; + }) + ); + } else { + definitions = [{ + def: openApiSpecDereferenced, + id: "", + }] + } const { actionName, ...otherActionData } = actionData; const { serverURL } = dataSourceConfig; From ed4f40be6df258dc81b5e38ec5fbdfe09deff288 Mon Sep 17 00:00:00 2001 From: "Tyler B. Thrailkill" Date: Fri, 8 Mar 2024 16:15:44 -0700 Subject: [PATCH 2/3] Pull definition logic into a new function --- .../node-service/src/plugins/openApi/index.ts | 40 ++++++++++--------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/server/node-service/src/plugins/openApi/index.ts b/server/node-service/src/plugins/openApi/index.ts index eaec12aac..265b23b44 100644 --- a/server/node-service/src/plugins/openApi/index.ts +++ b/server/node-service/src/plugins/openApi/index.ts @@ -50,18 +50,18 @@ interface ActionDataType { [key: string]: any; } -export async function runOpenApi( - actionData: ActionDataType, - dataSourceConfig: DataSourceDataType, +async function getDefinitions( spec: OpenAPI.Document | MultiOpenApiSpecItem[], - defaultHeaders?: Record, openApiSpecDereferenced?: OpenAPI.Document, -) { - const specList = Array.isArray(spec) ? spec : [{ spec, id: "" }]; - let definitions; - - if (!openApiSpecDereferenced) { - definitions = await Promise.all( +): Promise<{ def: OpenAPI.Document; id: string }[]> { + if (openApiSpecDereferenced) { + return [{ + def: openApiSpecDereferenced, + id: "", + }] + } else { + const specList = Array.isArray(spec) ? spec : [{ spec, id: "" }]; + return await Promise.all( specList.map(async ({id, spec}) => { const deRefedSpec = await SwaggerParser.dereference(spec); return { @@ -70,20 +70,22 @@ export async function runOpenApi( }; }) ); - } else { - definitions = [{ - def: openApiSpecDereferenced, - id: "", - }] } +} + +export async function runOpenApi( + actionData: ActionDataType, + dataSourceConfig: DataSourceDataType, + spec: OpenAPI.Document | MultiOpenApiSpecItem[], + defaultHeaders?: Record, + openApiSpecDereferenced?: OpenAPI.Document, +) { const { actionName, ...otherActionData } = actionData; const { serverURL } = dataSourceConfig; - let definition: OpenAPI.Document | undefined; - let operation; - let realOperationId; + let operation, realOperationId, definition: OpenAPI.Document | undefined; - for (const { id, def } of definitions) { + for (const {id, def} of await getDefinitions(spec, openApiSpecDereferenced)) { const ret = findOperation(actionName, def, id); if (ret) { definition = def; From 275cf68eb73e80a43034e5d9a88431949cfe6f0b Mon Sep 17 00:00:00 2001 From: "Tyler B. Thrailkill" Date: Fri, 8 Mar 2024 16:16:39 -0700 Subject: [PATCH 3/3] Small amount of code cleanup --- server/node-service/src/plugins/openApi/index.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/server/node-service/src/plugins/openApi/index.ts b/server/node-service/src/plugins/openApi/index.ts index 265b23b44..c99bcc9a3 100644 --- a/server/node-service/src/plugins/openApi/index.ts +++ b/server/node-service/src/plugins/openApi/index.ts @@ -12,11 +12,10 @@ import { isFile, } from "./util"; import { badRequest } from "../../common/error"; -import { safeJsonParse } from "../../common/util"; import { OpenAPI, OpenAPIV2, OpenAPIV3 } from "openapi-types"; import _ from "lodash"; import { fetch } from "../../common/fetch"; -import { RequestInit, Response } from "node-fetch"; +import { RequestInit } from "node-fetch"; const dataSourceConfig = { type: "dataSource",