HTTP support jsonPath; System plugin support save file. (#2969)

* perf: system plugin auto save file * feat: http support jsonPath * fix: assistant response * reset milvus version * fix: textarea register * fix: global variable * delete tip * doc
labring · Oct 22, 2024 · 9b0706e · 9b0706e
1 parent 718108a
commit 9b0706e
Show file tree

Hide file tree

Showing 30 changed files with 374 additions and 163 deletions.
diff --git a/docSite/content/zh-cn/docs/development/docker.md b/docSite/content/zh-cn/docs/development/docker.md
@@ -35,6 +35,8 @@ weight: 707
 
 ### Milvus版本
 
+暂不推荐，部分系统存在精度丢失，等待修复。
+
 对于千万级以上向量性能更优秀。
 
 [点击查看 Milvus 官方推荐配置](https://milvus.io/docs/prerequisite-docker.md)
@@ -49,6 +51,8 @@ weight: 707
 
 ### zilliz cloud版本
 
+暂不推荐，部分系统存在精度丢失，等待修复。
+
 亿级以上向量首选。
 
 由于向量库使用了 Cloud，无需占用本地资源，无需太关注。

diff --git a/docSite/content/zh-cn/docs/development/upgrading/4812.md b/docSite/content/zh-cn/docs/development/upgrading/4812.md
@@ -7,18 +7,23 @@ toc: true
 weight: 812
 ---
 
+## 更新指南
+
+
 ## 更新说明
 
 1. 新增 - 全局变量支持数字类型，支持配置默认值和部分输入框参数。
 2. 新增 - 插件自定义输入，文本输入框、数字输入框、选择框、开关，默认都支持作为变量引用。
-3. 新增 - FE_DOMAIN 环境变量，配置该环境变量后，上传文件/图片会补全后缀后得到完整地址。（可解决 docx 文件图片链接，有时会无法被模型识别问题）
-4. 新增 - 工具调用支持交互模式
+3. 新增 - FE_DOMAIN 环境变量，配置该环境变量后，上传文件/图片会补全后缀后得到完整地址。（可解决 docx 文件图片链接，有时模型会伪造图片域名）
+4. 新增 - 工具调用支持使用交互节点
 5. 新增 - Debug 模式支持输入全局变量
 6. 新增 - chat OpenAPI 文档
 7. 新增 - wiki 搜索插件
 8. 新增 - Google 搜索插件
 9. 新增 - 数据库连接和操作插件
 10. 新增 - Cookie 隐私协议提示
-11. 修复 - 文件后缀判断，去除 query 影响。
-12. 修复 - AI 响应为空时，会造成 LLM  历史记录合并。
-13. 修复 - 用户交互节点未阻塞流程。
+11. 新增 - HTTP 节点支持 JSONPath 表达式
+12. 修复 - 文件后缀判断，去除 query 影响。
+13. 修复 - AI 响应为空时，会造成 LLM  历史记录合并。
+14. 修复 - 用户交互节点未阻塞流程。
+15. 修复 - 新建 APP，有时候会导致空指针报错。
diff --git a/docSite/content/zh-cn/docs/workflow/modules/http.md b/docSite/content/zh-cn/docs/workflow/modules/http.md
@@ -120,10 +120,12 @@ HTTP 模块会向对应的地址发送一个 `HTTP` 请求，实际操作与 Pos
 
 ### 如何获取返回值
 
-从图中可以看出，FastGPT可以添加多个返回值，这个返回值并不代表接口的返回值，而是代表`如何解析接口返回值`，可以通过 key 来`提取`接口响应的值。例如: 
+从图中可以看出，FastGPT可以添加多个返回值，这个返回值并不代表接口的返回值，而是代表`如何解析接口返回值`，可以通过 `JSON path` 的语法，来`提取`接口响应的值。
+
+语法可以参考: https://github.com/JSONPath-Plus/JSONPath?tab=readme-ov-file
 
 {{< tabs tabTotal="2" >}}
-{{< tab tabName="接口响应格式" >}}
+{{< tab tabName="接口响应示例" >}}
 {{< markdownify >}}
 
 ```json
@@ -148,23 +150,23 @@ HTTP 模块会向对应的地址发送一个 `HTTP` 请求，实际操作与 Pos
 
 {{< /markdownify >}}
 {{< /tab >}}
-{{< tab tabName="FastGPT 转化后的格式" >}}
+{{< tab tabName="提取示例" >}}
 {{< markdownify >}}
 
 ```json
 {
-  "message": "测试",
-  "data.user": { "name": "xxx", "age": 12 },
-  "data.user.name": "xxx",
-  "data.user.age": 12,
-  "data.list": [ { "name": "xxx", "age": 50 }, [{ "test": 22 }] ],
-  "data.list[0]": { "name": "xxx", "age": 50 },
-  "data.list[0].name": "xxx",
-  "data.list[0].age": 50,
-  "data.list[1]": [ { "test": 22 } ],
-  "data.list[1][0]": { "test": 22 },
-  "data.list[1][0].test": 22,
-  "data.psw": "xxx"
+  "$.message": "测试",
+  "$.data.user": { "name": "xxx", "age": 12 },
+  "$.data.user.name": "xxx",
+  "$.data.user.age": 12,
+  "$.data.list": [ { "name": "xxx", "age": 50 }, [{ "test": 22 }] ],
+  "$.data.list[0]": { "name": "xxx", "age": 50 },
+  "$.data.list[0].name": "xxx",
+  "$.data.list[0].age": 50,
+  "$.data.list[1]": [ { "test": 22 } ],
+  "$.data.list[1][0]": { "test": 22 },
+  "$.data.list[1][0].test": 22,
+  "$.data.psw": "xxx"
 }
 ```
 
@@ -250,4 +252,4 @@ export default async function (ctx: FunctionContext) {
 
 - [谷歌搜索](/docs/workflow/examples/google_search/)
 - [发送飞书webhook](/docs/workflow/examples/feishu_webhook/)
-- [实验室预约（操作数据库）](/docs/workflow/examples/lab_appointment/)
+- [实验室预约（操作数据库）](/docs/workflow/examples/lab_appointment/)
diff --git a/packages/global/core/workflow/template/system/http468.ts b/packages/global/core/workflow/template/system/http468.ts
@@ -114,7 +114,9 @@ export const HttpNode468: FlowNodeTemplateType = {
   ],
   outputs: [
     {
-      ...Output_Template_AddOutput
+      ...Output_Template_AddOutput,
+      label: i18nT('workflow:http_extract_output'),
+      description: i18nT('workflow:http_extract_output_description')
     },
     {
       id: NodeOutputKeyEnum.error,

diff --git a/packages/plugins/package.json b/packages/plugins/package.json
@@ -10,6 +10,7 @@
     "expr-eval": "^2.0.2",
     "lodash": "^4.17.21",
     "mysql2": "^3.11.3",
+    "json5": "^2.2.3",
     "pg": "^8.10.0",
     "wikijs": "^6.4.1"
   },

diff --git a/packages/plugins/src/drawing/baseChart/index.ts b/packages/plugins/src/drawing/baseChart/index.ts
@@ -1,4 +1,8 @@
 import * as echarts from 'echarts';
+import json5 from 'json5';
+import { getFileSavePath } from '../../../utils';
+import * as fs from 'fs';
+import { SystemPluginSpecialResponse } from '../../../type.d';
 
 type Props = {
   title: string;
@@ -8,7 +12,7 @@ type Props = {
 };
 
 type Response = Promise<{
-  result: string;
+  result: SystemPluginSpecialResponse;
 }>;
 
 type SeriesData = {
@@ -37,8 +41,8 @@ const generateChart = async (title: string, xAxis: string, yAxis: string, chartT
   let parsedXAxis: string[] = [];
   let parsedYAxis: number[] = [];
   try {
-    parsedXAxis = JSON.parse(xAxis);
-    parsedYAxis = JSON.parse(yAxis);
+    parsedXAxis = json5.parse(xAxis);
+    parsedYAxis = json5.parse(yAxis);
   } catch (error: any) {
     console.error('解析数据时出错:', error);
     return Promise.reject('Data error');
@@ -78,16 +82,26 @@ const generateChart = async (title: string, xAxis: string, yAxis: string, chartT
 
   chart.setOption(option);
   // 生成 Base64 图像
-  const base64Image = chart.getDataURL({ type: 'png' });
+  const base64Image = chart.getDataURL();
+  const svgData = decodeURIComponent(base64Image.split(',')[1]);
+
+  const fileName = `chart_${Date.now()}.svg`;
+  const filePath = getFileSavePath(fileName);
+  fs.writeFileSync(filePath, svgData);
   // 释放图表实例
   chart.dispose();
 
-  return base64Image;
+  return filePath;
 };
 
 const main = async ({ title, xAxis, yAxis, chartType }: Props): Response => {
+  const filePath = await generateChart(title, xAxis, yAxis, chartType);
   return {
-    result: await generateChart(title, xAxis, yAxis, chartType)
+    result: {
+      type: 'SYSTEM_PLUGIN_FILE',
+      path: filePath,
+      contentType: 'image/svg+xml'
+    }
   };
 };
 

diff --git a/packages/plugins/src/drawing/baseChart/template.json b/packages/plugins/src/drawing/baseChart/template.json
@@ -50,10 +50,10 @@
             "canEdit": true,
             "key": "xAxis",
             "label": "xAxis",
-            "description": "x轴数据，例如：['A', 'B', 'C']",
+            "description": "x轴数据，例如：[\"A\", \"B\", \"C\"]",
             "defaultValue": "",
             "required": true,
-            "toolDescription": "x轴数据，例如：['A', 'B', 'C']",
+            "toolDescription": "x轴数据，例如：[\"A\", \"B\", \"C\"]",
             "list": [
               {
                 "label": "",

diff --git a/packages/plugins/type.d.ts b/packages/plugins/type.d.ts
@@ -1,7 +1,13 @@
 import { PluginTemplateType } from '@fastgpt/global/core/plugin/type.d';
+import { systemPluginResponseEnum } from '@fastgpt/global/core/workflow/runtime/constants';
 import { SystemPluginTemplateItemType } from '@fastgpt/global/core/workflow/type';
 
 export type SystemPluginResponseType = Promise<Record<string, any>>;
+export type SystemPluginSpecialResponse = {
+  type: 'SYSTEM_PLUGIN_FILE';
+  path: string;
+  contentType: string;
+};
 
 declare global {
   var systemPlugins: SystemPluginTemplateItemType[];

diff --git a/packages/plugins/utils.ts b/packages/plugins/utils.ts
@@ -0,0 +1,15 @@
+import path from 'path';
+import * as fs from 'fs';
+
+const isProduction = process.env.NODE_ENV === 'production';
+
+export const getFileSavePath = (name: string) => {
+  if (isProduction) {
+    return `/app/plugin_file/${name}`;
+  }
+  const filePath = path.join(process.cwd(), 'local', 'plugin_file', name);
+
+  fs.mkdirSync(path.dirname(filePath), { recursive: true });
+
+  return filePath;
+};
diff --git a/packages/service/common/string/tiktoken/index.ts b/packages/service/common/string/tiktoken/index.ts
@@ -25,7 +25,7 @@ export const countGptMessagesTokens = async (
       number
     >({
       name: WorkerNameEnum.countGptMessagesTokens,
-      maxReservedThreads: global.systemEnv?.tokenWorkers || 20
+      maxReservedThreads: global.systemEnv?.tokenWorkers || 50
     });
 
     const total = await workerController.run({ messages, tools, functionCall });

diff --git a/packages/service/common/vectorStore/milvus/class.ts b/packages/service/common/vectorStore/milvus/class.ts
@@ -24,10 +24,7 @@ export class MilvusCtrl {
 
     global.milvusClient = new MilvusClient({
       address: MILVUS_ADDRESS,
-      token: MILVUS_TOKEN,
-      loaderOptions: {
-        longs: Function
-      }
+      token: MILVUS_TOKEN
     });
 
     addLog.info(`Milvus connected`);
@@ -329,19 +326,10 @@ export class MilvusCtrl {
       id: string;
       teamId: string;
       datasetId: string;
-      int64: {
-        low: bigint;
-        high: bigint;
-        unsigned: boolean;
-      };
     }[];
 
     return rows.map((item) => ({
-      id: String({
-        low: BigInt(item.int64.low),
-        high: BigInt(item.int64.high),
-        unsigned: item.int64.unsigned
-      }),
+      id: String(item.id),
       teamId: item.teamId,
       datasetId: item.datasetId
     }));

diff --git a/packages/service/core/chat/utils.ts b/packages/service/core/chat/utils.ts
@@ -222,10 +222,9 @@ export const loadRequestMessages = async ({
             };
           }
         }
-        // if (item.role === ChatCompletionRequestMessageRoleEnum.Assistant) {
-        //   if (item.content === undefined && !item.tool_calls && !item.function_call) return;
-        //   if (Array.isArray(item.content) && item.content.length === 0) return;
-        // }
+        if (item.role === ChatCompletionRequestMessageRoleEnum.Assistant) {
+          if (item.content === undefined && !item.tool_calls && !item.function_call) return;
+        }
 
         return item;
       })