[Bug] [Storage] The Linkis result set should support the return of null

linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/io/FsPath.java

@@ -23,6 +23,8 @@
 import java.io.IOException;
 import java.net.URI;
 import java.net.URISyntaxException;
+import java.nio.file.FileSystems;
+import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.nio.file.attribute.PosixFilePermission;
 import java.nio.file.attribute.PosixFilePermissions;
@@ -168,6 +170,10 @@ public File toFile() {
     return new File(uri);
   }
 
+  public Path toPath() {
+    return FileSystems.getDefault().getPath(uri.toString());
+  }
+
   public String getUriString() {
     return uri.toString();
   }

linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/utils/ByteTimeUtils.java

@@ -213,7 +213,6 @@ private static long parseByteString(String str, ByteUnit unit) {
       } else {
         throw new NumberFormatException("Failed to parse byte string: " + str);
       }
-      suffix = suffix.toLowerCase();
       // Check for invalid suffixes
       if (suffix != null && !byteSuffixes.containsKey(suffix)) {
         throw new NumberFormatException("Invalid suffix: \"" + suffix + "\"");
@@ -297,6 +296,18 @@ public static long byteStringAsGb(String str) {
     return parseByteString(str, ByteUnit.GiB);
   }
 
+  /**
+   * Convert a passed byte string (e.g. -50b, -100k, or -250m) to gibibytes for internal use.
+   *
+   * <p>If no suffix is provided, the passed number is assumed to be in gibibytes.
+   */
+  public static long negativeByteStringAsGb(String str) {
+    if (str.startsWith("-")) {
+      return Math.negateExact(parseByteString(str.substring(1), ByteUnit.GiB));
+    }
+    return parseByteString(str, ByteUnit.GiB);
+  }
+
   /**
    * Returns a byte array with the buffer's contents, trying to avoid copying the data if possible.
    */

linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/utils/ResultSetUtils.java

@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.linkis.common.utils;
+
+import org.apache.linkis.common.io.FsPath;
+
+import java.io.File;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class ResultSetUtils {
+
+  // Sort in ASC order by numx in the result set _numx.dolphin file name
+  public static Comparator<FsPath> getResultSetFileComparatorOrderByNameNum() {
+
+    Comparator<FsPath> comparator =
+        (o1, o2) -> {
+          // get the num of file name
+          String regx = "\\d+";
+
+          String[] res1 = o1.getPath().split(File.separator);
+          String fileName1 = res1[res1.length - 1];
+          Matcher matcher1 = Pattern.compile(regx).matcher(fileName1);
+          int num1 = matcher1.find() ? Integer.parseInt(matcher1.group()) : Integer.MAX_VALUE;
+
+          String[] res2 = o2.getPath().split(File.separator);
+          String fileName2 = res2[res2.length - 1];
+          Matcher matcher2 = Pattern.compile(regx).matcher(fileName2);
+          int num2 = matcher2.find() ? Integer.parseInt(matcher2.group()) : Integer.MAX_VALUE;
+
+          return num1 - num2;
+        };
+    return comparator;
+  }
+
+  public static void sortByNameNum(List<FsPath> fsPathList) {
+    Collections.sort(fsPathList, getResultSetFileComparatorOrderByNameNum());
+  }
+}

linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/utils/VariableOperationUtils.java

@@ -28,7 +28,10 @@
 import java.util.Iterator;
 import java.util.Map;
 
+import com.fasterxml.jackson.databind.DeserializationFeature;
 import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.json.JsonMapper;
 import com.fasterxml.jackson.databind.node.ArrayNode;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 
@@ -49,6 +52,9 @@ public class VariableOperationUtils {
   private static final String[] CYCLES =
       new String[] {CYCLE_YEAR, CYCLE_MONTH, CYCLE_DAY, CYCLE_HOUR, CYCLE_MINUTE, CYCLE_SECOND};
 
+  private static final ObjectMapper mapper =
+      JsonMapper.builder().enable(DeserializationFeature.FAIL_ON_TRAILING_TOKENS).build();
+
   /**
    * yyyy-MM-dd HH:mm:ss
    *
@@ -78,30 +84,44 @@ public static ZonedDateTime toZonedDateTime(Date date) {
    * @param str
    * @return
    */
+  @Deprecated
   public static String replaces(ZonedDateTime dateTime, String str)
       throws VariableOperationFailedException {
-    return replaces(dateTime, str, true);
+    try {
+      JsonNode rootNode = mapper.readTree(str);
+      if (rootNode.isArray() || rootNode.isObject()) {
+        replaceJson(dateTime, rootNode);
+        return rootNode.toString();
+      }
+    } catch (Exception e) {
+      return replace(dateTime, str);
+    }
+    return replace(dateTime, str);
   }
 
   /**
    * json support variable operation
    *
+   * @param codeType
    * @param dateTime
    * @param str
-   * @param format
    * @return
    */
-  public static String replaces(ZonedDateTime dateTime, String str, boolean format)
+  public static String replaces(String codeType, ZonedDateTime dateTime, String str)
       throws VariableOperationFailedException {
-    try {
-      JsonNode rootNode = JsonUtils.jackson().readTree(str);
-      if (rootNode.isArray() || rootNode.isObject()) {
-        replaceJson(dateTime, rootNode);
-        return rootNode.toString();
+    String languageType = CodeAndRunTypeUtils.getLanguageTypeByCodeType(codeType, "");
+    if (languageType.equals(CodeAndRunTypeUtils.LANGUAGE_TYPE_JSON())) {
+      try {
+        JsonNode rootNode = mapper.readTree(str);
+        if (rootNode.isArray() || rootNode.isObject()) {
+          replaceJson(dateTime, rootNode);
+          return rootNode.toString();
+        }
+      } catch (Exception e) {
+        return replace(dateTime, str);
       }
-    } catch (Exception e) {
-      return replace(dateTime, str);
     }
+
     return replace(dateTime, str);
   }
 
@@ -197,8 +217,7 @@ private static void replaceJson(ZonedDateTime dateTime, JsonNode object)
         } else if (temp.isObject()) {
           replaceJson(dateTime, temp);
         } else {
-          arrayNode.remove(i);
-          arrayNode.insert(i, replace(dateTime, temp.toString()));
+          arrayNode.set(i, replace(dateTime, temp.toString()));
         }
       }
     } else if (object.isObject()) {

linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/utils/VariableUtils.scala

@@ -143,9 +143,10 @@ object VariableUtils extends Logging {
     }
     initAllDateVars(run_date, nameAndType)
     val codeOperation = parserVar(code, nameAndType)
-    parserDate(codeOperation, run_date)
+    parserDate(codeType, codeOperation, run_date)
   }
 
+  @deprecated
   private def parserDate(code: String, run_date: CustomDateType): String = {
     if (Configuration.VARIABLE_OPERATION) {
       val zonedDateTime: ZonedDateTime = VariableOperationUtils.toZonedDateTime(run_date.getDate)
@@ -155,6 +156,15 @@ object VariableUtils extends Logging {
     }
   }
 
+  private def parserDate(codeType: String, code: String, run_date: CustomDateType): String = {
+    if (Configuration.VARIABLE_OPERATION) {
+      val zonedDateTime: ZonedDateTime = VariableOperationUtils.toZonedDateTime(run_date.getDate)
+      VariableOperationUtils.replaces(codeType, zonedDateTime, code)
+    } else {
+      code
+    }
+  }
+
   private def initAllDateVars(
       run_date: CustomDateType,
       nameAndType: mutable.Map[String, variable.VariableType]
@@ -337,7 +347,7 @@ object VariableUtils extends Logging {
    *
    * @param code
    *   :code
-   * @param codeType
+   * @param languageType
    *   :SQL,PYTHON
    * @return
    */
@@ -346,27 +356,37 @@ object VariableUtils extends Logging {
 
     var varString: String = null
     var errString: String = null
+    var rightVarString: String = null
 
     languageType match {
       case CodeAndRunTypeUtils.LANGUAGE_TYPE_SQL =>
         varString = """\s*--@set\s*.+\s*"""
+        rightVarString = """^\s*--@set\s*.+\s*"""
         errString = """\s*--@.*"""
       case CodeAndRunTypeUtils.LANGUAGE_TYPE_PYTHON | CodeAndRunTypeUtils.LANGUAGE_TYPE_SHELL =>
         varString = """\s*#@set\s*.+\s*"""
+        rightVarString = """^\s*#@set\s*.+\s*"""
         errString = """\s*#@"""
       case CodeAndRunTypeUtils.LANGUAGE_TYPE_SCALA =>
         varString = """\s*//@set\s*.+\s*"""
+        rightVarString = """^\s*//@set\s*.+\s*"""
         errString = """\s*//@.+"""
       case CodeAndRunTypeUtils.LANGUAGE_TYPE_JAVA =>
         varString = """\s*!!@set\s*.+\s*"""
+        rightVarString = """^\s*!!@set\s*.+\s*"""
       case _ =>
         return nameAndValue
     }
 
     val customRegex = varString.r.unanchored
+    val customRightRegex = rightVarString.r.unanchored
     val errRegex = errString.r.unanchored
     code.split("\n").foreach { str =>
       {
+
+        if (customRightRegex.unapplySeq(str).size < customRegex.unapplySeq(str).size) {
+          logger.warn(s"code:$str is wrong custom variable format!!!")
+        }
         str match {
           case customRegex() =>
             val clearStr = if (str.endsWith(";")) str.substring(0, str.length - 1) else str