Merge pull request #4954 from WeDataSphere/master-pes

[Feature]New user configuration function added to the management console #4948

linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/conf/BDPConfiguration.scala

@@ -232,19 +232,20 @@ private[conf] object BDPConfiguration extends Logging {
 
   private[common] def formatValue[T](defaultValue: T, value: Option[String]): Option[T] = {
     if (value.isEmpty || value.exists(StringUtils.isEmpty)) return Option(defaultValue)
+    val trimValue = value.map(_.trim)
     val formattedValue = defaultValue match {
-      case _: String => value
-      case _: Byte => value.map(_.toByte)
-      case _: Short => value.map(_.toShort)
-      case _: Char => value.map(_.toCharArray.apply(0))
-      case _: Int => value.map(_.toInt)
-      case _: Long => value.map(_.toLong)
-      case _: Float => value.map(_.toFloat)
-      case _: Double => value.map(_.toDouble)
-      case _: Boolean => value.map(_.toBoolean)
-      case _: TimeType => value.map(new TimeType(_))
-      case _: ByteType => value.map(new ByteType(_))
-      case null => value
+      case _: String => trimValue
+      case _: Byte => trimValue.map(_.toByte)
+      case _: Short => trimValue.map(_.toShort)
+      case _: Char => trimValue.map(_.toCharArray.apply(0))
+      case _: Int => trimValue.map(_.toInt)
+      case _: Long => trimValue.map(_.toLong)
+      case _: Float => trimValue.map(_.toFloat)
+      case _: Double => trimValue.map(_.toDouble)
+      case _: Boolean => trimValue.map(_.toBoolean)
+      case _: TimeType => trimValue.map(new TimeType(_))
+      case _: ByteType => trimValue.map(new ByteType(_))
+      case null => trimValue
     }
     formattedValue.asInstanceOf[Option[T]]
   }

linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/log/LogUtils.scala

@@ -33,7 +33,7 @@ object LogUtils {
   }
 
   def generateERROR(rawLog: String): String = {
-    getTimeFormat + " " + "ERROR" + " " + rawLog
+    getTimeFormat + " " + ERROR_STR + " " + rawLog
   }
 
   def generateWarn(rawLog: String): String = {
@@ -52,4 +52,6 @@ object LogUtils {
     getTimeFormat + " " + "SYSTEM-WARN" + " " + rawLog
   }
 
+  val ERROR_STR = "ERROR"
+
 }

linkis-commons/linkis-protocol/src/main/scala/org/apache/linkis/protocol/utils/TaskUtils.scala

@@ -45,6 +45,14 @@ object TaskUtils {
       }
     } else params.put(key, waitToAdd)
 
+  private def clearMap(params: util.Map[String, AnyRef], key: String): Unit =
+    if (params != null && params.containsKey(key)) {
+      params.get(key) match {
+        case map: util.Map[String, AnyRef] => map.clear()
+        case _ => params.put(key, new util.HashMap[String, AnyRef]())
+      }
+    }
+
   private def getConfigurationMap(
       params: util.Map[String, AnyRef],
       key: String
@@ -84,13 +92,20 @@ object TaskUtils {
   def addStartupMap(params: util.Map[String, AnyRef], startupMap: util.Map[String, AnyRef]): Unit =
     addConfigurationMap(params, startupMap, TaskConstant.PARAMS_CONFIGURATION_STARTUP)
 
+  def clearStartupMap(params: util.Map[String, AnyRef]): Unit = {
+    val configurationMap = getMap(params, TaskConstant.PARAMS_CONFIGURATION)
+    if (!configurationMap.isEmpty) {
+      clearMap(configurationMap, TaskConstant.PARAMS_CONFIGURATION_STARTUP)
+    }
+  }
+
   def addRuntimeMap(params: util.Map[String, AnyRef], runtimeMap: util.Map[String, AnyRef]): Unit =
     addConfigurationMap(params, runtimeMap, TaskConstant.PARAMS_CONFIGURATION_RUNTIME)
 
   def addSpecialMap(params: util.Map[String, AnyRef], specialMap: util.Map[String, AnyRef]): Unit =
     addConfigurationMap(params, specialMap, TaskConstant.PARAMS_CONFIGURATION_SPECIAL)
 
-  // tdoo
+  // todo
   def getLabelsMap(params: util.Map[String, AnyRef]): util.Map[String, AnyRef] =
     getMap(params, TaskConstant.LABELS)
 

linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/utils/FileSystemUtils.java

@@ -96,6 +96,30 @@ public static void createNewFileWithFileSystem(
     }
   }
 
+  /**
+   * create new file and set file owner by FileSystem
+   *
+   * @param fileSystem
+   * @param filePath
+   * @param user
+   * @param createParentWhenNotExists
+   */
+  public static void createNewFileAndSetOwnerWithFileSystem(
+      FileSystem fileSystem, FsPath filePath, String user, boolean createParentWhenNotExists)
+      throws Exception {
+    if (!fileSystem.exists(filePath)) {
+      if (!fileSystem.exists(filePath.getParent())) {
+        if (!createParentWhenNotExists) {
+          throw new IOException(
+              "parent dir " + filePath.getParent().getPath() + " dose not exists.");
+        }
+        mkdirs(fileSystem, filePath.getParent(), user);
+      }
+      fileSystem.createNewFile(filePath);
+      fileSystem.setOwner(filePath, user);
+    }
+  }
+
   /**
    * Recursively create a directory
    *
@@ -133,4 +157,39 @@ public static boolean mkdirs(FileSystem fileSystem, FsPath dest, String user) th
     }
     return true;
   }
+
+  /**
+   * Recursively create a directory(递归创建目录) add owner info
+   *
+   * @param fileSystem
+   * @param dest
+   * @param user
+   * @throws IOException
+   * @return
+   */
+  public static boolean mkdirsAndSetOwner(FileSystem fileSystem, FsPath dest, String user)
+      throws IOException {
+    FsPath parentPath = dest.getParent();
+    Stack<FsPath> dirsToMake = new Stack<>();
+    dirsToMake.push(dest);
+    while (!fileSystem.exists(parentPath)) {
+      dirsToMake.push(parentPath);
+
+      if (Objects.isNull(parentPath.getParent())) {
+        // parent path of root is null
+        break;
+      }
+
+      parentPath = parentPath.getParent();
+    }
+    if (!fileSystem.canExecute(parentPath)) {
+      throw new IOException("You have not permission to access path " + dest.getPath());
+    }
+    while (!dirsToMake.empty()) {
+      FsPath path = dirsToMake.pop();
+      fileSystem.mkdir(path);
+      fileSystem.setOwner(path, user);
+    }
+    return true;
+  }
 }

linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/interactive/InteractiveJobDescBuilder.java

@@ -26,6 +26,7 @@
 import org.apache.linkis.cli.application.operator.ujes.LinkisJobOper;
 import org.apache.linkis.cli.application.operator.ujes.UJESClientFactory;
 import org.apache.linkis.cli.application.utils.CliUtils;
+import org.apache.linkis.cli.application.utils.LoggerManager;
 
 import org.apache.commons.lang3.StringUtils;
 
@@ -135,14 +136,22 @@ public static InteractiveJobDesc build(CliCtx ctx) {
     }
 
     if (StringUtils.isBlank(code) && StringUtils.isNotBlank(codePath)) {
-      code = CliUtils.readFile(codePath);
+      try {
+        code = CliUtils.readFile(codePath);
+      } catch (Exception e) {
+        LoggerManager.getInformationLogger().error("Failed to read file", e);
+        throw e;
+      }
     }
 
     executionMap.put(LinkisKeys.KEY_CODE, code);
     labelMap.put(LinkisKeys.KEY_ENGINETYPE, engineType);
     labelMap.put(LinkisKeys.KEY_CODETYPE, runType);
     labelMap.put(LinkisKeys.KEY_USER_CREATOR, proxyUsr + "-" + creator);
     sourceMap.put(LinkisKeys.KEY_SCRIPT_PATH, scriptPath);
+    if (ctx.getExtraMap().containsKey(CliKeys.VERSION)) {
+      sourceMap.put(LinkisKeys.CLI_VERSION, ctx.getExtraMap().get(CliKeys.VERSION));
+    }
     runtimeMap.put(LinkisKeys.KEY_HIVE_RESULT_DISPLAY_TBALE, true);
 
     desc.setCreator(creator);

linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/present/file/ResultFileWriter.java

@@ -32,7 +32,6 @@ public static void writeToFile(
       String pathName, String fileName, String content, Boolean overWrite) {
 
     File dir = new File(pathName);
-    File file = new File(fileName);
 
     if (!dir.exists()) {
       try {
@@ -47,6 +46,8 @@ public static void writeToFile(
       }
     }
 
+    File file = new File(dir.getAbsolutePath() + File.separator + fileName);
+
     if (overWrite || !file.exists()) {
       try {
         file.createNewFile();

linkis-computation-governance/linkis-computation-governance-common/src/main/java/org/apache/linkis/governance/common/entity/TemplateConfKey.java

@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.linkis.governance.common.entity;
+
+public class TemplateConfKey {
+
+  private String templateUuid;
+
+  private String key;
+
+  private String templateName;
+
+  private String configValue;
+
+  public String getTemplateUuid() {
+    return templateUuid;
+  }
+
+  public void setTemplateUuid(String templateUuid) {
+    this.templateUuid = templateUuid;
+  }
+
+  public String getKey() {
+    return key;
+  }
+
+  public void setKey(String key) {
+    this.key = key;
+  }
+
+  public String getTemplateName() {
+    return templateName;
+  }
+
+  public void setTemplateName(String templateName) {
+    this.templateName = templateName;
+  }
+
+  public String getConfigValue() {
+    return configValue;
+  }
+
+  public void setConfigValue(String configValue) {
+    this.configValue = configValue;
+  }
+
+  @Override
+  public String toString() {
+    return "TemplateKey{"
+        + "templateUuid='"
+        + templateUuid
+        + '\''
+        + ", key='"
+        + key
+        + '\''
+        + ", templateName='"
+        + templateName
+        + '\''
+        + ", configValue='"
+        + configValue
+        + '\''
+        + '}';
+  }
+}

linkis-computation-governance/linkis-computation-governance-common/src/main/java/org/apache/linkis/governance/common/protocol/conf/TemplateConfRequest.java

@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.linkis.governance.common.protocol.conf;
+
+import org.apache.linkis.protocol.message.RequestProtocol;
+
+public class TemplateConfRequest implements RequestProtocol {
+
+  private String templateUuid;
+
+  private String templateName;
+
+  public TemplateConfRequest(String templateUuid, String templateName) {
+    this.templateUuid = templateUuid;
+    this.templateName = templateName;
+  }
+
+  public TemplateConfRequest(String templateUuid) {
+    this.templateUuid = templateUuid;
+  }
+
+  public String getTemplateUuid() {
+    return templateUuid;
+  }
+
+  public void setTemplateUuid(String templateUuid) {
+    this.templateUuid = templateUuid;
+  }
+
+  public String getTemplateName() {
+    return templateName;
+  }
+
+  public void setTemplateName(String templateName) {
+    this.templateName = templateName;
+  }
+}

linkis-computation-governance/linkis-computation-governance-common/src/main/java/org/apache/linkis/governance/common/protocol/conf/TemplateConfResponse.java

@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.linkis.governance.common.protocol.conf;
+
+import org.apache.linkis.governance.common.entity.TemplateConfKey;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class TemplateConfResponse {
+
+  private List<TemplateConfKey> list = new ArrayList<>();
+
+  public List<TemplateConfKey> getList() {
+    return list;
+  }
+
+  public void setList(List<TemplateConfKey> list) {
+    this.list = list;
+  }
+}

linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/conf/GovernanceCommonConf.scala

@@ -90,10 +90,4 @@ object GovernanceCommonConf {
   val EC_APP_MANAGE_MODE =
     CommonVars("linkis.ec.app.manage.mode", "attach")
 
-  val SCALA_PARSE_APPEND_CODE_ENABLED =
-    CommonVars("linkis.scala.parse.append.code.enable", true).getValue
-
-  val SCALA_PARSE_APPEND_CODE =
-    CommonVars("linkis.scala.parse.append.code", "val linkisVar=1").getValue
-
 }

linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/exception/GovernanceErrorException.scala

@@ -17,7 +17,7 @@
 
 package org.apache.linkis.governance.common.exception
 
-import org.apache.linkis.common.exception.{ErrorException, ExceptionLevel, LinkisRuntimeException}
+import org.apache.linkis.common.exception.{ExceptionLevel, LinkisRuntimeException}
 
 class GovernanceErrorException(errorCode: Int, errorMsg: String)
     extends LinkisRuntimeException(errorCode, errorMsg) {