Dev 1.1.16 webank streamis fix code review

docs/configuration/linkis-storage.md

@@ -29,4 +29,4 @@
 |linkis-storage|wds.linkis.fs.hdfs.impl.disable.cache|  false |disable.cache |true|
 |linkis-storage|wds.linkis.hdfs.rest.errs|  |rest.errs|true|
 |linkis-storage|wds.linkis.resultset.row.max.str | 2m  | max.str |true|
-|linkis-storage|wds.linkis.storage.file.type | dolphin,sql,scala,py,hql,python,out,log,text,sh,jdbc,ngql,psql,fql,tsql | file.type |true|
+|linkis-storage|wds.linkis.storage.file.type | dolphin,sql,scala,py,hql,python,out,log,text,sh,jdbc,ngql,psql,fql,tsql,txt | file.type |true|

docs/errorcode/linkis-engineconn-plugin-core-errorcode.md

@@ -3,7 +3,7 @@
 | module name(模块名) | error code(错误码)  | describe(描述) |enumeration name(枚举)| Exception Class(类名)|
 | -------- | -------- | ----- |-----|-----|
 |linkis-engineconn-plugin-core |10001|Failed to createEngineConnLaunchRequest(创建 EngineConnLaunchRequest失败)|FAILED_CREATE_ELR|EngineconnCoreErrorCodeSummary|
-|linkis-engineconn-plugin-core |10001|The engine plug-in material is abnormal, please check whether the material is uploaded successfully(引擎插件物料异常，请检查物料是否上传成功)|EN_PLUGIN_MATERIAL_SOURCE_EXCEPTION|EngineconnCoreErrorCodeSummary|
+|linkis-engineconn-plugin-core |10001|The engine plugin material is abnormal, please check whether the material is uploaded successfully(引擎插件物料异常，请检查物料是否上传成功)|EN_PLUGIN_MATERIAL_SOURCE_EXCEPTION|EngineconnCoreErrorCodeSummary|
 |linkis-engineconn-plugin-core |10001|EngineTypeLabel are requested(需要参数 EngineTypeLabel)|ETL_REQUESTED|EngineconnCoreErrorCodeSummary|
 |linkis-engineconn-plugin-core |20000|Cannot instance EngineConnExecution(无法实例化 EngineConnExecution)|CANNOT_INSTANCE_ECE|EngineconnCoreErrorCodeSummary|
 |linkis-engineconn-plugin-core |20000|Cannot find default ExecutorFactory(找不到默认的 ExecutorFactory)|CANNOT_DEFAULT_EF|EngineconnCoreErrorCodeSummary|

docs/errorcode/python-errorcode.md

@@ -1,7 +1,7 @@
 ## python errorcode
 
-| 模块名(服务名) | 错误码  | 描述 |enumeration name| Exception Class|
-| -------- | -------- | ----- |-----|-----|
-|python|41001| |PYTHON_EXECUTE_ERROR|LinkisPythonErrorCodeSummary|
-|python|60003|Pyspark process  has stopped, query failed!(Pyspark 进程已停止，查询失败！)|PYSPARK_PROCESSS_STOPPED|LinkisPythonErrorCodeSummary|
-|python|400201|Invalid python session.(无效的 python 会话.)|INVALID_PYTHON_SESSION|LinkisPythonErrorCodeSummary|
+| 模块名(服务名) | 错误码    | 描述 |enumeration name| Exception Class|
+| -------- |--------| ----- |-----|-----|
+|python| 60002  | |PYTHON_EXECUTE_ERROR|LinkisPythonErrorCodeSummary|
+|python| 60003  |Pyspark process  has stopped, query failed!(Pyspark 进程已停止，查询失败！)|PYSPARK_PROCESSS_STOPPED|LinkisPythonErrorCodeSummary|
+|python| 400201 |Invalid python session.(无效的 python 会话.)|INVALID_PYTHON_SESSION|LinkisPythonErrorCodeSummary|

docs/info-1.3.2.md

@@ -4,4 +4,5 @@
 |------------------| ----- |----------------------------------------------------------------------|------| ------------------------------------------------------- |
 | linkis-jobhistory | 新增  | wds.linkis.jobhistory.admin | hadoop |可以查看所有历史任务的用户 注意：wds.linkis.governance.station.admin 为管理用户（也具有可以查看所有历史任务的权限）|
 | linkis | 新增  | wds.linkis.governance.station.admin.token |   /具有管理员权限的特殊token|
+| linkis | 新增  | linkis.configuration.remove.application.cache | IDE |清除该应用的配置缓存|
 | cg-entrance | 新增  | linkis.entrance.auto.clean.dirty.data.enable | true |entrance重启调用ps-jobhistory接口是否开启，ture为开启，取值范围：true或false|

linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/utils/ByteTimeUtils.java

@@ -213,7 +213,6 @@ private static long parseByteString(String str, ByteUnit unit) {
       } else {
         throw new NumberFormatException("Failed to parse byte string: " + str);
       }
-      suffix = suffix.toLowerCase();
       // Check for invalid suffixes
       if (suffix != null && !byteSuffixes.containsKey(suffix)) {
         throw new NumberFormatException("Invalid suffix: \"" + suffix + "\"");
@@ -297,6 +296,18 @@ public static long byteStringAsGb(String str) {
     return parseByteString(str, ByteUnit.GiB);
   }
 
+  /**
+   * Convert a passed byte string (e.g. -50b, -100k, or -250m) to gibibytes for internal use.
+   *
+   * <p>If no suffix is provided, the passed number is assumed to be in gibibytes.
+   */
+  public static long negativeByteStringAsGb(String str) {
+    if (str.startsWith("-")) {
+      return Math.negateExact(parseByteString(str.substring(1), ByteUnit.GiB));
+    }
+    return parseByteString(str, ByteUnit.GiB);
+  }
+
   /**
    * Returns a byte array with the buffer's contents, trying to avoid copying the data if possible.
    */

linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/utils/ResultSetUtils.java

@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.linkis.common.utils;
+
+import org.apache.linkis.common.io.FsPath;
+
+import java.io.File;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class ResultSetUtils {
+
+  // Sort in ASC order by numx in the result set _numx.dolphin file name
+  public static Comparator<FsPath> getResultSetFileComparatorOrderByNameNum() {
+
+    Comparator<FsPath> comparator =
+        (o1, o2) -> {
+          // get the num of file name
+          String regx = "\\d+";
+
+          String[] res1 = o1.getPath().split(File.separator);
+          String fileName1 = res1[res1.length - 1];
+          Matcher matcher1 = Pattern.compile(regx).matcher(fileName1);
+          int num1 = matcher1.find() ? Integer.parseInt(matcher1.group()) : Integer.MAX_VALUE;
+
+          String[] res2 = o2.getPath().split(File.separator);
+          String fileName2 = res2[res2.length - 1];
+          Matcher matcher2 = Pattern.compile(regx).matcher(fileName2);
+          int num2 = matcher2.find() ? Integer.parseInt(matcher2.group()) : Integer.MAX_VALUE;
+
+          return num1 - num2;
+        };
+    return comparator;
+  }
+
+  public static void sortByNameNum(List<FsPath> fsPathList) {
+    Collections.sort(fsPathList, getResultSetFileComparatorOrderByNameNum());
+  }
+}

linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/utils/VariableOperationUtils.java

@@ -28,7 +28,10 @@
 import java.util.Iterator;
 import java.util.Map;
 
+import com.fasterxml.jackson.databind.DeserializationFeature;
 import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.json.JsonMapper;
 import com.fasterxml.jackson.databind.node.ArrayNode;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 
@@ -49,6 +52,9 @@ public class VariableOperationUtils {
   private static final String[] CYCLES =
       new String[] {CYCLE_YEAR, CYCLE_MONTH, CYCLE_DAY, CYCLE_HOUR, CYCLE_MINUTE, CYCLE_SECOND};
 
+  private static final ObjectMapper mapper =
+      JsonMapper.builder().enable(DeserializationFeature.FAIL_ON_TRAILING_TOKENS).build();
+
   /**
    * yyyy-MM-dd HH:mm:ss
    *
@@ -78,30 +84,44 @@ public static ZonedDateTime toZonedDateTime(Date date) {
    * @param str
    * @return
    */
+  @Deprecated
   public static String replaces(ZonedDateTime dateTime, String str)
       throws VariableOperationFailedException {
-    return replaces(dateTime, str, true);
+    try {
+      JsonNode rootNode = mapper.readTree(str);
+      if (rootNode.isArray() || rootNode.isObject()) {
+        replaceJson(dateTime, rootNode);
+        return rootNode.toString();
+      }
+    } catch (Exception e) {
+      return replace(dateTime, str);
+    }
+    return replace(dateTime, str);
   }
 
   /**
    * json support variable operation
    *
+   * @param codeType
    * @param dateTime
    * @param str
-   * @param format
    * @return
    */
-  public static String replaces(ZonedDateTime dateTime, String str, boolean format)
+  public static String replaces(String codeType, ZonedDateTime dateTime, String str)
       throws VariableOperationFailedException {
-    try {
-      JsonNode rootNode = JsonUtils.jackson().readTree(str);
-      if (rootNode.isArray() || rootNode.isObject()) {
-        replaceJson(dateTime, rootNode);
-        return rootNode.toString();
+    String languageType = CodeAndRunTypeUtils.getLanguageTypeByCodeType(codeType, "");
+    if (languageType.equals(CodeAndRunTypeUtils.LANGUAGE_TYPE_JSON())) {
+      try {
+        JsonNode rootNode = mapper.readTree(str);
+        if (rootNode.isArray() || rootNode.isObject()) {
+          replaceJson(dateTime, rootNode);
+          return rootNode.toString();
+        }
+      } catch (Exception e) {
+        return replace(dateTime, str);
       }
-    } catch (Exception e) {
-      return replace(dateTime, str);
     }
+
     return replace(dateTime, str);
   }
 
@@ -197,7 +217,7 @@ private static void replaceJson(ZonedDateTime dateTime, JsonNode object)
         } else if (temp.isObject()) {
           replaceJson(dateTime, temp);
         } else {
-          arrayNode.insert(i, replace(dateTime, temp.toString()));
+          arrayNode.set(i, replace(dateTime, temp.toString()));
         }
       }
     } else if (object.isObject()) {

linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/conf/BDPConfiguration.scala

@@ -22,7 +22,7 @@ import org.apache.linkis.common.utils.{Logging, Utils}
 import org.apache.commons.io.IOUtils
 import org.apache.commons.lang3.StringUtils
 
-import java.io.{File, FileInputStream, InputStream, IOException}
+import java.io._
 import java.util.Properties
 import java.util.concurrent.TimeUnit
 import java.util.concurrent.locks.ReentrantReadWriteLock
@@ -140,15 +140,20 @@ private[conf] object BDPConfiguration extends Logging {
 
   private def initConfig(config: Properties, filePath: String) {
     var inputStream: InputStream = null
-
+    var reader: InputStreamReader = null
+    var buff: BufferedReader = null
     Utils.tryFinally {
       Utils.tryCatch {
         inputStream = new FileInputStream(filePath)
-        config.load(inputStream)
+        reader = new InputStreamReader(inputStream, "UTF-8")
+        buff = new BufferedReader(reader)
+        config.load(buff)
       } { case e: IOException =>
         logger.error("Can't load " + filePath, e)
       }
     } {
+      IOUtils.closeQuietly(buff)
+      IOUtils.closeQuietly(reader)
       IOUtils.closeQuietly(inputStream)
     }
   }

linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/utils/VariableUtils.scala

@@ -143,9 +143,10 @@ object VariableUtils extends Logging {
     }
     initAllDateVars(run_date, nameAndType)
     val codeOperation = parserVar(code, nameAndType)
-    parserDate(codeOperation, run_date)
+    parserDate(codeType, codeOperation, run_date)
   }
 
+  @deprecated
   private def parserDate(code: String, run_date: CustomDateType): String = {
     if (Configuration.VARIABLE_OPERATION) {
       val zonedDateTime: ZonedDateTime = VariableOperationUtils.toZonedDateTime(run_date.getDate)
@@ -155,6 +156,15 @@ object VariableUtils extends Logging {
     }
   }
 
+  private def parserDate(codeType: String, code: String, run_date: CustomDateType): String = {
+    if (Configuration.VARIABLE_OPERATION) {
+      val zonedDateTime: ZonedDateTime = VariableOperationUtils.toZonedDateTime(run_date.getDate)
+      VariableOperationUtils.replaces(codeType, zonedDateTime, code)
+    } else {
+      code
+    }
+  }
+
   private def initAllDateVars(
       run_date: CustomDateType,
       nameAndType: mutable.Map[String, variable.VariableType]
@@ -337,7 +347,7 @@ object VariableUtils extends Logging {
    *
    * @param code
    *   :code
-   * @param codeType
+   * @param languageType
    *   :SQL,PYTHON
    * @return
    */
@@ -346,27 +356,37 @@ object VariableUtils extends Logging {
 
     var varString: String = null
     var errString: String = null
+    var rightVarString: String = null
 
     languageType match {
       case CodeAndRunTypeUtils.LANGUAGE_TYPE_SQL =>
         varString = """\s*--@set\s*.+\s*"""
+        rightVarString = """^\s*--@set\s*.+\s*"""
         errString = """\s*--@.*"""
       case CodeAndRunTypeUtils.LANGUAGE_TYPE_PYTHON | CodeAndRunTypeUtils.LANGUAGE_TYPE_SHELL =>
         varString = """\s*#@set\s*.+\s*"""
+        rightVarString = """^\s*#@set\s*.+\s*"""
         errString = """\s*#@"""
       case CodeAndRunTypeUtils.LANGUAGE_TYPE_SCALA =>
         varString = """\s*//@set\s*.+\s*"""
+        rightVarString = """^\s*//@set\s*.+\s*"""
         errString = """\s*//@.+"""
       case CodeAndRunTypeUtils.LANGUAGE_TYPE_JAVA =>
         varString = """\s*!!@set\s*.+\s*"""
+        rightVarString = """^\s*!!@set\s*.+\s*"""
       case _ =>
         return nameAndValue
     }
 
     val customRegex = varString.r.unanchored
+    val customRightRegex = rightVarString.r.unanchored
     val errRegex = errString.r.unanchored
     code.split("\n").foreach { str =>
       {
+
+        if (customRightRegex.unapplySeq(str).size < customRegex.unapplySeq(str).size) {
+          logger.warn(s"code:$str is wrong custom variable format!!!")
+        }
         str match {
           case customRegex() =>
             val clearStr = if (str.endsWith(";")) str.substring(0, str.length - 1) else str

linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/present/PresentModeImplTest.java → linkis-commons/linkis-common/src/test/java/org/apache/linkis/common/conf/BDPConfigurationTest.java

@@ -15,22 +15,27 @@
  * limitations under the License.
  */
 
-package org.apache.linkis.cli.core.present;
+package org.apache.linkis.common.conf;
 
 import org.junit.jupiter.api.Assertions;
-import org.junit.jupiter.api.DisplayName;
 import org.junit.jupiter.api.Test;
 
-public class PresentModeImplTest {
+/** BDPConfiguration Tester */
+public class BDPConfigurationTest {
 
   @Test
-  @DisplayName("enumTest")
-  public void enumTest() {
+  public void testGetOption() {
 
-    String stdoutName = PresentModeImpl.STDOUT.getName();
-    String textFileName = PresentModeImpl.TEXT_FILE.getName();
+    Assertions.assertEquals(
+        "properties支持中文",
+        BDPConfiguration.getOption(
+                CommonVars.apply("linkis.jobhistory.error.msg.tip", "properties支持中文"))
+            .get());
 
-    Assertions.assertEquals("STDOUT", stdoutName);
-    Assertions.assertEquals("TEXT_FILE", textFileName);
+    Assertions.assertEquals(
+        "properties支持中文(默认)",
+        BDPConfiguration.getOption(
+                CommonVars.apply("linkis.jobhistory.error.msg.tip1", "properties支持中文(默认)"))
+            .get());
   }
 }