[WIP]feat: update some part of linkis-web in linkis-web-next

.github/workflows/integration-test.yml

@@ -45,7 +45,7 @@ jobs:
       TAG: ${{ github.sha }}
       SKIP_TEST: true
       HUB: ghcr.io/apache/linkis
-      LINKIS_VERSION: 1.5.0-SNAPSHOT
+      LINKIS_VERSION: 1.4.0-SNAPSHOT
     steps:
       - name: Free up disk space
         run: |

.github/workflows/publish-docker.yaml

@@ -34,7 +34,7 @@ jobs:
       TAG: ${{ github.sha }}
       SKIP_TEST: true
       HUB: ghcr.io/apache/linkis
-      LINKIS_VERSION: 1.5.0-SNAPSHOT
+      LINKIS_VERSION: 1.4.0-SNAPSHOT
     steps:
     - name: Checkout
       uses: actions/checkout@v2

docs/configuration/linkis-instance-label-client.md

@@ -1,5 +1,5 @@
-## linkis-pes-rpc-client 配置
+## linkis-instance-label-client 配置
 
-| 模块名(服务名)              | 参数名 | 默认值 | 描述 |
-|-----------------------| -------- | ----- |----- |
-| linkis-pes-rpc-client |wds.linkis.instance.label.server.name|linkis-ps-publicservice|instance.label.server.name|
+| 模块名(服务名) | 参数名 | 默认值 | 描述 |
+| -------- | -------- | ----- |----- |
+| linkis-instance-label-client |wds.linkis.instance.label.server.name|linkis-ps-publicservice|instance.label.server.name|

docs/configuration/spark.md

@@ -3,7 +3,6 @@
 
 | Module Name (Service Name) | Parameter Name | Default Value | Description |Used|
 | -------- | -------- | ----- |----- |  -----   |
-|spark|linkis.spark.yarn.cluster.jars|hdfs:///spark/cluster|spark.yarn.cluster.jars|
 |spark|linkis.spark.etl.support.hudi|false|spark.etl.support.hudi|
 |spark|linkis.bgservice.store.prefix|hdfs:///tmp/bdp-ide/|bgservice.store.prefix|
 |spark|linkis.bgservice.store.suffix|  |bgservice.store.suffix|
@@ -28,11 +27,6 @@
 |spark|wds.linkis.spark.engineconn.fatal.log|error writing class;OutOfMemoryError|spark.engineconn.fatal.log|
 |spark|wds.linkis.spark.engine.scala.replace_package_header.enable| true |spark.engine.scala.replace_package_header.enable|
 
-Use spark yarn cluster mode,need to set label "engingeConnRuntimeMode": "yarnCluster",and need to upload the dependence of the spark to 'linkis.spark.yarn.cluster.jar'(the default value is 'hdfs:///spark/cluster')
-spark dependencies include jars and configuration files，For example: '/appcom/Install/linkis/lib/linkis-engineconn-plugins/spark/dist/3.2.1/lib/*.jar','/appcom/Install/linkis/conf/*'
-
-Precautions for using yarnCluster:
-Eureka url if 127.0.0.1 should be changed to the real host, such as "127.0.0.1:20303/eureka/" should be changed to "wds001:20303/eureka/"
 
 The spark-excel package may cause class conflicts,need to download separately,put it in spark lib
 wget https://repo1.maven.org/maven2/com/crealytics/spark-excel-2.12.17-3.2.2_2.12/3.2.2_0.18.1/spark-excel-2.12.17-3.2.2_2.12-3.2.2_0.18.1.jar

docs/errorcode/linkis-bml-client.md

@@ -1,9 +1,9 @@
-## linkis-pes-client errorcode
+## linkis-bml-client errorcode
 
 
-| 模块名(服务名)          | 错误码   | 描述 | enumeration name(枚举)| Exception Class(类名)|
-|-------------------|-------| ----- |---------|---------|
-| linkis-pes-client | 20060 |the result returned by the repository client POST request does not match(物料库客户端POST请求返回的result不匹配)|POST_REQUEST_RESULT_NOT_MATCH|BmlClientErrorCodeSummary|
-| linkis-pes-client | 20061 |failed to copy inputStream and outputStream (inputStream和outputStream流copy失败)|BML_CLIENT_FAILED|BmlClientErrorCodeSummary|
-| linkis-pes-client | 20062 |serverUrl cannot be null(serverUrl 不能为空)|SERVER_URL_NOT_NULL|BmlClientErrorCodeSummary|
+| 模块名(服务名) | 错误码   | 描述 | enumeration name(枚举)| Exception Class(类名)|
+| -------- |-------| ----- |---------|---------|
+|linkis-bml-client| 20060 |the result returned by the repository client POST request does not match(物料库客户端POST请求返回的result不匹配)|POST_REQUEST_RESULT_NOT_MATCH|BmlClientErrorCodeSummary|
+|linkis-bml-client| 20061 |failed to copy inputStream and outputStream (inputStream和outputStream流copy失败)|BML_CLIENT_FAILED|BmlClientErrorCodeSummary|
+|linkis-bml-client| 20062 |serverUrl cannot be null(serverUrl 不能为空)|SERVER_URL_NOT_NULL|BmlClientErrorCodeSummary|
 

docs/errorcode/linkis-cs-client-errorcode.md

@@ -1,16 +1,16 @@
-## linkis-pes-client   errorcode
+## linkis-cs-client   errorcode
 
-| module name(模块名)  | error code(错误码)  | describe(描述) |enumeration name(枚举)| Exception Class(类名)|
-|-------------------| -------- | ----- |-----|-----|
-| linkis-pes-client |80015|create context failed(创建上下文失败)|CREATE_CONTEXT_FAILED|CsClientErrorCodeSummary|
-| linkis-pes-client  |80015|get context value failed(获取上下文值失败)|GET_CONTEXT_VALUE_FAILED|CsClientErrorCodeSummary|
-| linkis-pes-client  |80015|update context failed(更新上下文失败)|UPDATE_CONTEXT_FAILED|CsClientErrorCodeSummary|
-| linkis-pes-client  |80015|reset context failed(重置上下文失败)|RESET_CONTEXT_FAILED|CsClientErrorCodeSummary|
-| linkis-pes-client  |80015|remove context failed(刪除上下文失败)|REMOVE_CONTEXT_FAILED|CsClientErrorCodeSummary|
-| linkis-pes-client  |80015|bind context id failed(绑定上下文 ID 失败)|BIND_CONTEXTID_FAILED|CsClientErrorCodeSummary|
-| linkis-pes-client  |80015|search condition failed(搜索失败)|SEARCH_CONDITION_FAILED|CsClientErrorCodeSummary|
-| linkis-pes-client  |80015|execute  failed(执行失败)|EXECUTE_FALIED|CsClientErrorCodeSummary|
-| linkis-pes-client  |80017|searchHAIDByTime failed(搜索HAIDByTime失败)|HAIDBYTIME_FAILED|CsClientErrorCodeSummary|
-| linkis-pes-client  |80017|batch Clear Context By HAID failed(通过 HAID 批量清除上下文失败)|CLEAR_CONTEXT_HAID_FAILED|CsClientErrorCodeSummary|
+| module name(模块名) | error code(错误码)  | describe(描述) |enumeration name(枚举)| Exception Class(类名)|
+| -------- | -------- | ----- |-----|-----|
+|linkis-cs-client  |80015|create context failed(创建上下文失败)|CREATE_CONTEXT_FAILED|CsClientErrorCodeSummary|
+|linkis-cs-client  |80015|get context value failed(获取上下文值失败)|GET_CONTEXT_VALUE_FAILED|CsClientErrorCodeSummary|
+|linkis-cs-client  |80015|update context failed(更新上下文失败)|UPDATE_CONTEXT_FAILED|CsClientErrorCodeSummary|
+|linkis-cs-client  |80015|reset context failed(重置上下文失败)|RESET_CONTEXT_FAILED|CsClientErrorCodeSummary|
+|linkis-cs-client  |80015|remove context failed(刪除上下文失败)|REMOVE_CONTEXT_FAILED|CsClientErrorCodeSummary|
+|linkis-cs-client  |80015|bind context id failed(绑定上下文 ID 失败)|BIND_CONTEXTID_FAILED|CsClientErrorCodeSummary|
+|linkis-cs-client  |80015|search condition failed(搜索失败)|SEARCH_CONDITION_FAILED|CsClientErrorCodeSummary|
+|linkis-cs-client  |80015|execute  failed(执行失败)|EXECUTE_FALIED|CsClientErrorCodeSummary|
+|linkis-cs-client  |80017|searchHAIDByTime failed(搜索HAIDByTime失败)|HAIDBYTIME_FAILED|CsClientErrorCodeSummary|
+|linkis-cs-client  |80017|batch Clear Context By HAID failed(通过 HAID 批量清除上下文失败)|CLEAR_CONTEXT_HAID_FAILED|CsClientErrorCodeSummary|
 
 

docs/errorcode/linkis-datasource-client-errorcode.md

@@ -1,16 +1,16 @@
-## linkis-pes-client   errorcode
+## linkis-datasource-client   errorcode
 
 | module name(模块名) | error code(错误码)  | describe(描述) |enumeration name(枚举)| Exception Class(类名)|
 | -------- | -------- | ----- |-----|-----|
-|linkis-pes-client  |31000|serverUrl cannot be null.(serverUrl 不能为空.)|SERVERURL_CANNOT_NULL|DatasourceClientErrorCodeSummary|
-|linkis-pes-client  |31000|version is needed!(版本为空！)|VERSION_NEEDED|DatasourceClientErrorCodeSummary|
-|linkis-pes-client  |31000|version is needed!(需要版本ID！)"|VERSIONID_NEEDED|DatasourceClientErrorCodeSummary|
-|linkis-pes-client  |31000|dataSourceId is needed!(需要dataSourceId！)|DATASOURCEID_NEEDED|DatasourceClientErrorCodeSummary|
-|linkis-pes-client  |31000|dataSourceName is needed!(需要dataSourceName！)|DATASOURCENAME_NEEDED|DatasourceClientErrorCodeSummary|
-|linkis-pes-client  |31000|user is needed!(用户为空！)|USER_NEEDED|DatasourceClientErrorCodeSummary|
-|linkis-pes-client  |31000|system is needed!(系统为空！)|SYSTEM_NEEDED|DatasourceClientErrorCodeSummary|
-|linkis-pes-client  |31000|Cannot encode the name of data source:{0} for request(无法对请求的数据源名称进行编码：{0})|CANNOT_SOURCE|DatasourceClientErrorCodeSummary|
-|linkis-pes-client  |31000|database is needed!(数据库为空！)|DATABASE_NEEDED|DatasourceClientErrorCodeSummary|
-|linkis-pes-client  |31000|table is needed!(表为空！)|TABLE_NEEDED|DatasourceClientErrorCodeSummary|
+|linkis-datasource-client  |31000|serverUrl cannot be null.(serverUrl 不能为空.)|SERVERURL_CANNOT_NULL|DatasourceClientErrorCodeSummary|
+|linkis-datasource-client  |31000|version is needed!(版本为空！)|VERSION_NEEDED|DatasourceClientErrorCodeSummary|
+|linkis-datasource-client  |31000|version is needed!(需要版本ID！)"|VERSIONID_NEEDED|DatasourceClientErrorCodeSummary|
+|linkis-datasource-client  |31000|dataSourceId is needed!(需要dataSourceId！)|DATASOURCEID_NEEDED|DatasourceClientErrorCodeSummary|
+|linkis-datasource-client  |31000|dataSourceName is needed!(需要dataSourceName！)|DATASOURCENAME_NEEDED|DatasourceClientErrorCodeSummary|
+|linkis-datasource-client  |31000|user is needed!(用户为空！)|USER_NEEDED|DatasourceClientErrorCodeSummary|
+|linkis-datasource-client  |31000|system is needed!(系统为空！)|SYSTEM_NEEDED|DatasourceClientErrorCodeSummary|
+|linkis-datasource-client  |31000|Cannot encode the name of data source:{0} for request(无法对请求的数据源名称进行编码：{0})|CANNOT_SOURCE|DatasourceClientErrorCodeSummary|
+|linkis-datasource-client  |31000|database is needed!(数据库为空！)|DATABASE_NEEDED|DatasourceClientErrorCodeSummary|
+|linkis-datasource-client  |31000|table is needed!(表为空！)|TABLE_NEEDED|DatasourceClientErrorCodeSummary|
 
 

docs/errorcode/linkis-instance-label-errorcode.md

@@ -1,7 +1,7 @@
 ## linkis-instance-label  errorcode
 
 | 模块名(服务名) | 错误码  | 描述 | Exception Class|
-| -------- | -------- | ----- |-----|
+|  | -------- | ----- |-----|
 |linkis-instance-label |14100|Failed to insert service instance(插入服务实例失败)|LinkisInstanceLabelErrorCodeSummary|
 |linkis-instance-label |14100|Only admin can view all instances(只有管理员才能查看所有实例).|LinkisInstanceLabelErrorCodeSummary|
 |linkis-instance-label |14100|Only admin can modify instance label(只有管理员才能修改标签).|LinkisInstanceLabelErrorCodeSummary|

linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/io/FsPath.java

@@ -23,6 +23,8 @@
 import java.io.IOException;
 import java.net.URI;
 import java.net.URISyntaxException;
+import java.nio.file.FileSystems;
+import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.nio.file.attribute.PosixFilePermission;
 import java.nio.file.attribute.PosixFilePermissions;
@@ -168,6 +170,10 @@ public File toFile() {
     return new File(uri);
   }
 
+  public Path toPath() {
+    return FileSystems.getDefault().getPath(uri.toString());
+  }
+
   public String getUriString() {
     return uri.toString();
   }

linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/utils/ByteTimeUtils.java

@@ -213,7 +213,6 @@ private static long parseByteString(String str, ByteUnit unit) {
       } else {
         throw new NumberFormatException("Failed to parse byte string: " + str);
       }
-      suffix = suffix.toLowerCase();
       // Check for invalid suffixes
       if (suffix != null && !byteSuffixes.containsKey(suffix)) {
         throw new NumberFormatException("Invalid suffix: \"" + suffix + "\"");
@@ -297,6 +296,18 @@ public static long byteStringAsGb(String str) {
     return parseByteString(str, ByteUnit.GiB);
   }
 
+  /**
+   * Convert a passed byte string (e.g. -50b, -100k, or -250m) to gibibytes for internal use.
+   *
+   * <p>If no suffix is provided, the passed number is assumed to be in gibibytes.
+   */
+  public static long negativeByteStringAsGb(String str) {
+    if (str.startsWith("-")) {
+      return Math.negateExact(parseByteString(str.substring(1), ByteUnit.GiB));
+    }
+    return parseByteString(str, ByteUnit.GiB);
+  }
+
   /**
    * Returns a byte array with the buffer's contents, trying to avoid copying the data if possible.
    */

linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/utils/ResultSetUtils.java

@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.linkis.common.utils;
+
+import org.apache.linkis.common.io.FsPath;
+
+import java.io.File;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class ResultSetUtils {
+
+  // Sort in ASC order by numx in the result set _numx.dolphin file name
+  public static Comparator<FsPath> getResultSetFileComparatorOrderByNameNum() {
+
+    Comparator<FsPath> comparator =
+        (o1, o2) -> {
+          // get the num of file name
+          String regx = "\\d+";
+
+          String[] res1 = o1.getPath().split(File.separator);
+          String fileName1 = res1[res1.length - 1];
+          Matcher matcher1 = Pattern.compile(regx).matcher(fileName1);
+          int num1 = matcher1.find() ? Integer.parseInt(matcher1.group()) : Integer.MAX_VALUE;
+
+          String[] res2 = o2.getPath().split(File.separator);
+          String fileName2 = res2[res2.length - 1];
+          Matcher matcher2 = Pattern.compile(regx).matcher(fileName2);
+          int num2 = matcher2.find() ? Integer.parseInt(matcher2.group()) : Integer.MAX_VALUE;
+
+          return num1 - num2;
+        };
+    return comparator;
+  }
+
+  public static void sortByNameNum(List<FsPath> fsPathList) {
+    Collections.sort(fsPathList, getResultSetFileComparatorOrderByNameNum());
+  }
+}

linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/utils/VariableOperationUtils.java

@@ -28,7 +28,10 @@
 import java.util.Iterator;
 import java.util.Map;
 
+import com.fasterxml.jackson.databind.DeserializationFeature;
 import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.json.JsonMapper;
 import com.fasterxml.jackson.databind.node.ArrayNode;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 
@@ -49,6 +52,9 @@ public class VariableOperationUtils {
   private static final String[] CYCLES =
       new String[] {CYCLE_YEAR, CYCLE_MONTH, CYCLE_DAY, CYCLE_HOUR, CYCLE_MINUTE, CYCLE_SECOND};
 
+  private static final ObjectMapper mapper =
+      JsonMapper.builder().enable(DeserializationFeature.FAIL_ON_TRAILING_TOKENS).build();
+
   /**
    * yyyy-MM-dd HH:mm:ss
    *
@@ -78,30 +84,44 @@ public static ZonedDateTime toZonedDateTime(Date date) {
    * @param str
    * @return
    */
+  @Deprecated
   public static String replaces(ZonedDateTime dateTime, String str)
       throws VariableOperationFailedException {
-    return replaces(dateTime, str, true);
+    try {
+      JsonNode rootNode = mapper.readTree(str);
+      if (rootNode.isArray() || rootNode.isObject()) {
+        replaceJson(dateTime, rootNode);
+        return rootNode.toString();
+      }
+    } catch (Exception e) {
+      return replace(dateTime, str);
+    }
+    return replace(dateTime, str);
   }
 
   /**
    * json support variable operation
    *
+   * @param codeType
    * @param dateTime
    * @param str
-   * @param format
    * @return
    */
-  public static String replaces(ZonedDateTime dateTime, String str, boolean format)
+  public static String replaces(String codeType, ZonedDateTime dateTime, String str)
       throws VariableOperationFailedException {
-    try {
-      JsonNode rootNode = JsonUtils.jackson().readTree(str);
-      if (rootNode.isArray() || rootNode.isObject()) {
-        replaceJson(dateTime, rootNode);
-        return rootNode.toString();
+    String languageType = CodeAndRunTypeUtils.getLanguageTypeByCodeType(codeType, "");
+    if (languageType.equals(CodeAndRunTypeUtils.LANGUAGE_TYPE_JSON())) {
+      try {
+        JsonNode rootNode = mapper.readTree(str);
+        if (rootNode.isArray() || rootNode.isObject()) {
+          replaceJson(dateTime, rootNode);
+          return rootNode.toString();
+        }
+      } catch (Exception e) {
+        return replace(dateTime, str);
       }
-    } catch (Exception e) {
-      return replace(dateTime, str);
     }
+
     return replace(dateTime, str);
   }
 
@@ -197,8 +217,7 @@ private static void replaceJson(ZonedDateTime dateTime, JsonNode object)
         } else if (temp.isObject()) {
           replaceJson(dateTime, temp);
         } else {
-          arrayNode.remove(i);
-          arrayNode.insert(i, replace(dateTime, temp.toString()));
+          arrayNode.set(i, replace(dateTime, temp.toString()));
         }
       }
     } else if (object.isObject()) {

linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/utils/VariableUtils.scala

@@ -143,9 +143,10 @@ object VariableUtils extends Logging {
     }
     initAllDateVars(run_date, nameAndType)
     val codeOperation = parserVar(code, nameAndType)
-    parserDate(codeOperation, run_date)
+    parserDate(codeType, codeOperation, run_date)
   }
 
+  @deprecated
   private def parserDate(code: String, run_date: CustomDateType): String = {
     if (Configuration.VARIABLE_OPERATION) {
       val zonedDateTime: ZonedDateTime = VariableOperationUtils.toZonedDateTime(run_date.getDate)
@@ -155,6 +156,15 @@ object VariableUtils extends Logging {
     }
   }
 
+  private def parserDate(codeType: String, code: String, run_date: CustomDateType): String = {
+    if (Configuration.VARIABLE_OPERATION) {
+      val zonedDateTime: ZonedDateTime = VariableOperationUtils.toZonedDateTime(run_date.getDate)
+      VariableOperationUtils.replaces(codeType, zonedDateTime, code)
+    } else {
+      code
+    }
+  }
+
   private def initAllDateVars(
       run_date: CustomDateType,
       nameAndType: mutable.Map[String, variable.VariableType]
@@ -337,7 +347,7 @@ object VariableUtils extends Logging {
    *
    * @param code
    *   :code
-   * @param codeType
+   * @param languageType
    *   :SQL,PYTHON
    * @return
    */
@@ -346,27 +356,37 @@ object VariableUtils extends Logging {
 
     var varString: String = null
     var errString: String = null
+    var rightVarString: String = null
 
     languageType match {
       case CodeAndRunTypeUtils.LANGUAGE_TYPE_SQL =>
         varString = """\s*--@set\s*.+\s*"""
+        rightVarString = """^\s*--@set\s*.+\s*"""
         errString = """\s*--@.*"""
       case CodeAndRunTypeUtils.LANGUAGE_TYPE_PYTHON | CodeAndRunTypeUtils.LANGUAGE_TYPE_SHELL =>
         varString = """\s*#@set\s*.+\s*"""
+        rightVarString = """^\s*#@set\s*.+\s*"""
         errString = """\s*#@"""
       case CodeAndRunTypeUtils.LANGUAGE_TYPE_SCALA =>
         varString = """\s*//@set\s*.+\s*"""
+        rightVarString = """^\s*//@set\s*.+\s*"""
         errString = """\s*//@.+"""
       case CodeAndRunTypeUtils.LANGUAGE_TYPE_JAVA =>
         varString = """\s*!!@set\s*.+\s*"""
+        rightVarString = """^\s*!!@set\s*.+\s*"""
       case _ =>
         return nameAndValue
     }
 
     val customRegex = varString.r.unanchored
+    val customRightRegex = rightVarString.r.unanchored
     val errRegex = errString.r.unanchored
     code.split("\n").foreach { str =>
       {
+
+        if (customRightRegex.unapplySeq(str).size < customRegex.unapplySeq(str).size) {
+          logger.warn(s"code:$str is wrong custom variable format!!!")
+        }
         str match {
           case customRegex() =>
             val clearStr = if (str.endsWith(";")) str.substring(0, str.length - 1) else str