diff --git a/docs/configuration/linkis-storage.md b/docs/configuration/linkis-storage.md index 45b07afc1d..8e04b073be 100644 --- a/docs/configuration/linkis-storage.md +++ b/docs/configuration/linkis-storage.md @@ -29,4 +29,4 @@ |linkis-storage|wds.linkis.fs.hdfs.impl.disable.cache| false |disable.cache |true| |linkis-storage|wds.linkis.hdfs.rest.errs| |rest.errs|true| |linkis-storage|wds.linkis.resultset.row.max.str | 2m | max.str |true| -|linkis-storage|wds.linkis.storage.file.type | dolphin,sql,scala,py,hql,python,out,log,text,sh,jdbc,ngql,psql,fql,tsql | file.type |true| +|linkis-storage|wds.linkis.storage.file.type | dolphin,sql,scala,py,hql,python,out,log,text,sh,jdbc,ngql,psql,fql,tsql,txt | file.type |true| diff --git a/docs/errorcode/linkis-engineconn-plugin-core-errorcode.md b/docs/errorcode/linkis-engineconn-plugin-core-errorcode.md index a929d062d2..0fc2521a76 100644 --- a/docs/errorcode/linkis-engineconn-plugin-core-errorcode.md +++ b/docs/errorcode/linkis-engineconn-plugin-core-errorcode.md @@ -3,7 +3,7 @@ | module name(模块名) | error code(错误码) | describe(描述) |enumeration name(枚举)| Exception Class(类名)| | -------- | -------- | ----- |-----|-----| |linkis-engineconn-plugin-core |10001|Failed to createEngineConnLaunchRequest(创建 EngineConnLaunchRequest失败)|FAILED_CREATE_ELR|EngineconnCoreErrorCodeSummary| -|linkis-engineconn-plugin-core |10001|The engine plug-in material is abnormal, please check whether the material is uploaded successfully(引擎插件物料异常，请检查物料是否上传成功)|EN_PLUGIN_MATERIAL_SOURCE_EXCEPTION|EngineconnCoreErrorCodeSummary| +|linkis-engineconn-plugin-core |10001|The engine plugin material is abnormal, please check whether the material is uploaded successfully(引擎插件物料异常，请检查物料是否上传成功)|EN_PLUGIN_MATERIAL_SOURCE_EXCEPTION|EngineconnCoreErrorCodeSummary| |linkis-engineconn-plugin-core |10001|EngineTypeLabel are requested(需要参数 EngineTypeLabel)|ETL_REQUESTED|EngineconnCoreErrorCodeSummary| |linkis-engineconn-plugin-core |20000|Cannot instance EngineConnExecution(无法实例化 EngineConnExecution)|CANNOT_INSTANCE_ECE|EngineconnCoreErrorCodeSummary| |linkis-engineconn-plugin-core |20000|Cannot find default ExecutorFactory(找不到默认的 ExecutorFactory)|CANNOT_DEFAULT_EF|EngineconnCoreErrorCodeSummary| diff --git a/docs/errorcode/python-errorcode.md b/docs/errorcode/python-errorcode.md index 8a4f843484..cf63aaa3fe 100644 --- a/docs/errorcode/python-errorcode.md +++ b/docs/errorcode/python-errorcode.md @@ -1,7 +1,7 @@ ## python errorcode -| 模块名(服务名) | 错误码 | 描述 |enumeration name| Exception Class| -| -------- | -------- | ----- |-----|-----| -|python|41001| |PYTHON_EXECUTE_ERROR|LinkisPythonErrorCodeSummary| -|python|60003|Pyspark process has stopped, query failed!(Pyspark 进程已停止，查询失败！)|PYSPARK_PROCESSS_STOPPED|LinkisPythonErrorCodeSummary| -|python|400201|Invalid python session.(无效的 python 会话.)|INVALID_PYTHON_SESSION|LinkisPythonErrorCodeSummary| +| 模块名(服务名) | 错误码 | 描述 |enumeration name| Exception Class| +| -------- |--------| ----- |-----|-----| +|python| 60002 | |PYTHON_EXECUTE_ERROR|LinkisPythonErrorCodeSummary| +|python| 60003 |Pyspark process has stopped, query failed!(Pyspark 进程已停止，查询失败！)|PYSPARK_PROCESSS_STOPPED|LinkisPythonErrorCodeSummary| +|python| 400201 |Invalid python session.(无效的 python 会话.)|INVALID_PYTHON_SESSION|LinkisPythonErrorCodeSummary| diff --git a/docs/info-1.3.2.md b/docs/info-1.3.2.md index 55e5aeecbc..0c690517fb 100644 --- a/docs/info-1.3.2.md +++ b/docs/info-1.3.2.md @@ -4,4 +4,5 @@ |------------------| ----- |----------------------------------------------------------------------|------| ------------------------------------------------------- | | linkis-jobhistory | 新增 | wds.linkis.jobhistory.admin | hadoop |可以查看所有历史任务的用户 注意：wds.linkis.governance.station.admin 为管理用户（也具有可以查看所有历史任务的权限）| | linkis | 新增 | wds.linkis.governance.station.admin.token | /具有管理员权限的特殊token| +| linkis | 新增 | linkis.configuration.remove.application.cache | IDE |清除该应用的配置缓存| | cg-entrance | 新增 | linkis.entrance.auto.clean.dirty.data.enable | true |entrance重启调用ps-jobhistory接口是否开启，ture为开启，取值范围：true或false| diff --git a/linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/utils/ByteTimeUtils.java b/linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/utils/ByteTimeUtils.java index d23f4a0867..0ecb3dc2a5 100644 --- a/linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/utils/ByteTimeUtils.java +++ b/linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/utils/ByteTimeUtils.java @@ -213,7 +213,6 @@ private static long parseByteString(String str, ByteUnit unit) { } else { throw new NumberFormatException("Failed to parse byte string: " + str); } - suffix = suffix.toLowerCase(); // Check for invalid suffixes if (suffix != null && !byteSuffixes.containsKey(suffix)) { throw new NumberFormatException("Invalid suffix: \"" + suffix + "\""); @@ -297,6 +296,18 @@ public static long byteStringAsGb(String str) { return parseByteString(str, ByteUnit.GiB); } + /** + * Convert a passed byte string (e.g. -50b, -100k, or -250m) to gibibytes for internal use. + * + *

If no suffix is provided, the passed number is assumed to be in gibibytes. + */ + public static long negativeByteStringAsGb(String str) { + if (str.startsWith("-")) { + return Math.negateExact(parseByteString(str.substring(1), ByteUnit.GiB)); + } + return parseByteString(str, ByteUnit.GiB); + } + /** * Returns a byte array with the buffer's contents, trying to avoid copying the data if possible. */ diff --git a/linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/utils/ResultSetUtils.java b/linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/utils/ResultSetUtils.java new file mode 100644 index 0000000000..a367b38b80 --- /dev/null +++ b/linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/utils/ResultSetUtils.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.common.utils; + +import org.apache.linkis.common.io.FsPath; + +import java.io.File; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class ResultSetUtils { + + // Sort in ASC order by numx in the result set _numx.dolphin file name + public static Comparator getResultSetFileComparatorOrderByNameNum() { + + Comparator comparator = + (o1, o2) -> { + // get the num of file name + String regx = "\\d+"; + + String[] res1 = o1.getPath().split(File.separator); + String fileName1 = res1[res1.length - 1]; + Matcher matcher1 = Pattern.compile(regx).matcher(fileName1); + int num1 = matcher1.find() ? Integer.parseInt(matcher1.group()) : Integer.MAX_VALUE; + + String[] res2 = o2.getPath().split(File.separator); + String fileName2 = res2[res2.length - 1]; + Matcher matcher2 = Pattern.compile(regx).matcher(fileName2); + int num2 = matcher2.find() ? Integer.parseInt(matcher2.group()) : Integer.MAX_VALUE; + + return num1 - num2; + }; + return comparator; + } + + public static void sortByNameNum(List fsPathList) { + Collections.sort(fsPathList, getResultSetFileComparatorOrderByNameNum()); + } +} diff --git a/linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/utils/VariableOperationUtils.java b/linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/utils/VariableOperationUtils.java index d71f8b40e6..615472474d 100644 --- a/linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/utils/VariableOperationUtils.java +++ b/linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/utils/VariableOperationUtils.java @@ -28,7 +28,10 @@ import java.util.Iterator; import java.util.Map; +import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.json.JsonMapper; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; @@ -49,6 +52,9 @@ public class VariableOperationUtils { private static final String[] CYCLES = new String[] {CYCLE_YEAR, CYCLE_MONTH, CYCLE_DAY, CYCLE_HOUR, CYCLE_MINUTE, CYCLE_SECOND}; + private static final ObjectMapper mapper = + JsonMapper.builder().enable(DeserializationFeature.FAIL_ON_TRAILING_TOKENS).build(); + /** * yyyy-MM-dd HH:mm:ss * @@ -78,30 +84,44 @@ public static ZonedDateTime toZonedDateTime(Date date) { * @param str * @return */ + @Deprecated public static String replaces(ZonedDateTime dateTime, String str) throws VariableOperationFailedException { - return replaces(dateTime, str, true); + try { + JsonNode rootNode = mapper.readTree(str); + if (rootNode.isArray() || rootNode.isObject()) { + replaceJson(dateTime, rootNode); + return rootNode.toString(); + } + } catch (Exception e) { + return replace(dateTime, str); + } + return replace(dateTime, str); } /** * json support variable operation * + * @param codeType * @param dateTime * @param str - * @param format * @return */ - public static String replaces(ZonedDateTime dateTime, String str, boolean format) + public static String replaces(String codeType, ZonedDateTime dateTime, String str) throws VariableOperationFailedException { - try { - JsonNode rootNode = JsonUtils.jackson().readTree(str); - if (rootNode.isArray() || rootNode.isObject()) { - replaceJson(dateTime, rootNode); - return rootNode.toString(); + String languageType = CodeAndRunTypeUtils.getLanguageTypeByCodeType(codeType, ""); + if (languageType.equals(CodeAndRunTypeUtils.LANGUAGE_TYPE_JSON())) { + try { + JsonNode rootNode = mapper.readTree(str); + if (rootNode.isArray() || rootNode.isObject()) { + replaceJson(dateTime, rootNode); + return rootNode.toString(); + } + } catch (Exception e) { + return replace(dateTime, str); } - } catch (Exception e) { - return replace(dateTime, str); } + return replace(dateTime, str); } @@ -197,7 +217,7 @@ private static void replaceJson(ZonedDateTime dateTime, JsonNode object) } else if (temp.isObject()) { replaceJson(dateTime, temp); } else { - arrayNode.insert(i, replace(dateTime, temp.toString())); + arrayNode.set(i, replace(dateTime, temp.toString())); } } } else if (object.isObject()) { diff --git a/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/conf/BDPConfiguration.scala b/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/conf/BDPConfiguration.scala index 55535e5336..14febab63a 100644 --- a/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/conf/BDPConfiguration.scala +++ b/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/conf/BDPConfiguration.scala @@ -22,7 +22,7 @@ import org.apache.linkis.common.utils.{Logging, Utils} import org.apache.commons.io.IOUtils import org.apache.commons.lang3.StringUtils -import java.io.{File, FileInputStream, InputStream, IOException} +import java.io._ import java.util.Properties import java.util.concurrent.TimeUnit import java.util.concurrent.locks.ReentrantReadWriteLock @@ -140,15 +140,20 @@ private[conf] object BDPConfiguration extends Logging { private def initConfig(config: Properties, filePath: String) { var inputStream: InputStream = null - + var reader: InputStreamReader = null + var buff: BufferedReader = null Utils.tryFinally { Utils.tryCatch { inputStream = new FileInputStream(filePath) - config.load(inputStream) + reader = new InputStreamReader(inputStream, "UTF-8") + buff = new BufferedReader(reader) + config.load(buff) } { case e: IOException => logger.error("Can't load " + filePath, e) } } { + IOUtils.closeQuietly(buff) + IOUtils.closeQuietly(reader) IOUtils.closeQuietly(inputStream) } } diff --git a/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/utils/VariableUtils.scala b/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/utils/VariableUtils.scala index 30bdeb4b14..6c5bd7cf3c 100644 --- a/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/utils/VariableUtils.scala +++ b/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/utils/VariableUtils.scala @@ -143,9 +143,10 @@ object VariableUtils extends Logging { } initAllDateVars(run_date, nameAndType) val codeOperation = parserVar(code, nameAndType) - parserDate(codeOperation, run_date) + parserDate(codeType, codeOperation, run_date) } + @deprecated private def parserDate(code: String, run_date: CustomDateType): String = { if (Configuration.VARIABLE_OPERATION) { val zonedDateTime: ZonedDateTime = VariableOperationUtils.toZonedDateTime(run_date.getDate) @@ -155,6 +156,15 @@ object VariableUtils extends Logging { } } + private def parserDate(codeType: String, code: String, run_date: CustomDateType): String = { + if (Configuration.VARIABLE_OPERATION) { + val zonedDateTime: ZonedDateTime = VariableOperationUtils.toZonedDateTime(run_date.getDate) + VariableOperationUtils.replaces(codeType, zonedDateTime, code) + } else { + code + } + } + private def initAllDateVars( run_date: CustomDateType, nameAndType: mutable.Map[String, variable.VariableType] @@ -337,7 +347,7 @@ object VariableUtils extends Logging { * * @param code * :code - * @param codeType + * @param languageType * :SQL,PYTHON * @return */ @@ -346,27 +356,37 @@ object VariableUtils extends Logging { var varString: String = null var errString: String = null + var rightVarString: String = null languageType match { case CodeAndRunTypeUtils.LANGUAGE_TYPE_SQL => varString = """\s*--@set\s*.+\s*""" + rightVarString = """^\s*--@set\s*.+\s*""" errString = """\s*--@.*""" case CodeAndRunTypeUtils.LANGUAGE_TYPE_PYTHON | CodeAndRunTypeUtils.LANGUAGE_TYPE_SHELL => varString = """\s*#@set\s*.+\s*""" + rightVarString = """^\s*#@set\s*.+\s*""" errString = """\s*#@""" case CodeAndRunTypeUtils.LANGUAGE_TYPE_SCALA => varString = """\s*//@set\s*.+\s*""" + rightVarString = """^\s*//@set\s*.+\s*""" errString = """\s*//@.+""" case CodeAndRunTypeUtils.LANGUAGE_TYPE_JAVA => varString = """\s*!!@set\s*.+\s*""" + rightVarString = """^\s*!!@set\s*.+\s*""" case _ => return nameAndValue } val customRegex = varString.r.unanchored + val customRightRegex = rightVarString.r.unanchored val errRegex = errString.r.unanchored code.split("\n").foreach { str => { + + if (customRightRegex.unapplySeq(str).size < customRegex.unapplySeq(str).size) { + logger.warn(s"code:$str is wrong custom variable format!!!") + } str match { case customRegex() => val clearStr = if (str.endsWith(";")) str.substring(0, str.length - 1) else str diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/present/PresentModeImplTest.java b/linkis-commons/linkis-common/src/test/java/org/apache/linkis/common/conf/BDPConfigurationTest.java similarity index 60% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/present/PresentModeImplTest.java rename to linkis-commons/linkis-common/src/test/java/org/apache/linkis/common/conf/BDPConfigurationTest.java index 8f776bef81..5a025eb8b0 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/present/PresentModeImplTest.java +++ b/linkis-commons/linkis-common/src/test/java/org/apache/linkis/common/conf/BDPConfigurationTest.java @@ -15,22 +15,27 @@ * limitations under the License. */ -package org.apache.linkis.cli.core.present; +package org.apache.linkis.common.conf; import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; -public class PresentModeImplTest { +/** BDPConfiguration Tester */ +public class BDPConfigurationTest { @Test - @DisplayName("enumTest") - public void enumTest() { + public void testGetOption() { - String stdoutName = PresentModeImpl.STDOUT.getName(); - String textFileName = PresentModeImpl.TEXT_FILE.getName(); + Assertions.assertEquals( + "properties支持中文", + BDPConfiguration.getOption( + CommonVars.apply("linkis.jobhistory.error.msg.tip", "properties支持中文")) + .get()); - Assertions.assertEquals("STDOUT", stdoutName); - Assertions.assertEquals("TEXT_FILE", textFileName); + Assertions.assertEquals( + "properties支持中文(默认)", + BDPConfiguration.getOption( + CommonVars.apply("linkis.jobhistory.error.msg.tip1", "properties支持中文(默认)")) + .get()); } } diff --git a/linkis-commons/linkis-common/src/test/java/org/apache/linkis/common/utils/ByteTimeUtilsTest.java b/linkis-commons/linkis-common/src/test/java/org/apache/linkis/common/utils/ByteTimeUtilsTest.java new file mode 100644 index 0000000000..f548d89d46 --- /dev/null +++ b/linkis-commons/linkis-common/src/test/java/org/apache/linkis/common/utils/ByteTimeUtilsTest.java @@ -0,0 +1,175 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.common.utils; + +import java.util.function.Function; + +import com.google.common.collect.ImmutableMap; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +class ByteTimeUtilsTest { + + private static final ImmutableMap> opFunction = + ImmutableMap.>builder() + .put("byteStringAsBytes", tar -> ByteTimeUtils.byteStringAsBytes(tar)) + .put("byteStringAsKb", tar -> ByteTimeUtils.byteStringAsKb(tar)) + .put("byteStringAsMb", tar -> ByteTimeUtils.byteStringAsMb(tar)) + .put("byteStringAsGb", tar -> ByteTimeUtils.byteStringAsGb(tar)) + .build(); + + private static final ImmutableMap convertToByte = + ImmutableMap.builder() + .put("1", 1l) + .put("1b", 1l) + .put("1B", 1l) + .put("1k", 1024l) + .put("1K", 1024l) + .put("1kb", 1024l) + .put("1Kb", 1024l) + .put("1kB", 1024l) + .put("1KB", 1024l) + .put("1m", 1024l * 1024l) + .put("1M", 1024l * 1024l) + .put("1mb", 1024l * 1024l) + .put("1Mb", 1024l * 1024l) + .put("1mB", 1024l * 1024l) + .put("1MB", 1024l * 1024l) + .put("1g", 1024l * 1024l * 1024l) + .put("1G", 1024l * 1024l * 1024l) + .put("1gb", 1024l * 1024l * 1024l) + .put("1gB", 1024l * 1024l * 1024l) + .put("1Gb", 1024l * 1024l * 1024l) + .put("1GB", 1024l * 1024l * 1024l) + .put("1t", 1024l * 1024l * 1024l * 1024l) + .put("1T", 1024l * 1024l * 1024l * 1024l) + .put("1tb", 1024l * 1024l * 1024l * 1024l) + .put("1Tb", 1024l * 1024l * 1024l * 1024l) + .put("1tB", 1024l * 1024l * 1024l * 1024l) + .put("1TB", 1024l * 1024l * 1024l * 1024l) + .put("1p", 1024l * 1024l * 1024l * 1024l * 1024l) + .put("1P", 1024l * 1024l * 1024l * 1024l * 1024l) + .put("1pb", 1024l * 1024l * 1024l * 1024l * 1024l) + .put("1Pb", 1024l * 1024l * 1024l * 1024l * 1024l) + .put("1pB", 1024l * 1024l * 1024l * 1024l * 1024l) + .put("1PB", 1024l * 1024l * 1024l * 1024l * 1024l) + .build(); + + private static final ImmutableMap convertToKB = + ImmutableMap.builder() + .put("1", 1l) + .put("1024b", 1l) + .put("1024B", 1l) + .put("1k", 1l) + .put("1K", 1l) + .put("1kb", 1l) + .put("1Kb", 1l) + .put("1kB", 1l) + .put("1KB", 1l) + .put("1m", 1024l) + .put("1M", 1024l) + .put("1mb", 1024l) + .put("1Mb", 1024l) + .put("1mB", 1024l) + .put("1MB", 1024l) + .put("1g", 1024l * 1024l) + .put("1G", 1024l * 1024l) + .put("1gb", 1024l * 1024l) + .put("1gB", 1024l * 1024l) + .put("1Gb", 1024l * 1024l) + .put("1GB", 1024l * 1024l) + .build(); + + private static final ImmutableMap convertToMB = + ImmutableMap.builder() + .put("1", 1l) + .put("1024k", 1l) + .put("1024K", 1l) + .put("1024kb", 1l) + .put("1024Kb", 1l) + .put("1024kB", 1l) + .put("1024KB", 1l) + .put("1m", 1l) + .put("1M", 1l) + .put("1mb", 1l) + .put("1Mb", 1l) + .put("1mB", 1l) + .put("1MB", 1l) + .put("1g", 1024l) + .put("1G", 1024l) + .put("1gb", 1024l) + .put("1gB", 1024l) + .put("1Gb", 1024l) + .put("1GB", 1024l) + .build(); + + private static final ImmutableMap convertToGB = + ImmutableMap.builder() + .put("1", 1l) + .put("1024m", 1l) + .put("1024M", 1l) + .put("1024mb", 1l) + .put("1024Mb", 1l) + .put("1024mB", 1l) + .put("1024MB", 1l) + .put("1g", 1l) + .put("1G", 1l) + .put("1gb", 1l) + .put("1gB", 1l) + .put("1Gb", 1l) + .put("1GB", 1l) + .put("1t", 1024l) + .put("1T", 1024l) + .put("1tb", 1024l) + .put("1Tb", 1024l) + .put("1tB", 1024l) + .put("1TB", 1024l) + .build(); + + @Test + void byteStringAsBytes() { + convertToByte.forEach( + (k, v) -> Assertions.assertEquals(opFunction.get("byteStringAsBytes").apply(k), v)); + Assertions.assertThrows( + IllegalArgumentException.class, () -> opFunction.get("byteStringAsBytes").apply("1A")); + } + + @Test + void byteStringAsKb() { + convertToKB.forEach( + (k, v) -> Assertions.assertEquals(opFunction.get("byteStringAsKb").apply(k), v)); + Assertions.assertThrows( + IllegalArgumentException.class, () -> opFunction.get("byteStringAsKb").apply("1a")); + } + + @Test + void byteStringAsMb() { + convertToMB.forEach( + (k, v) -> Assertions.assertEquals(opFunction.get("byteStringAsMb").apply(k), v)); + Assertions.assertThrows( + IllegalArgumentException.class, () -> opFunction.get("byteStringAsMb").apply("1c")); + } + + @Test + void byteStringAsGb() { + convertToGB.forEach( + (k, v) -> Assertions.assertEquals(opFunction.get("byteStringAsGb").apply(k), v)); + Assertions.assertThrows( + IllegalArgumentException.class, () -> opFunction.get("byteStringAsGb").apply("1C")); + } +} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/resources/conf/user.properties b/linkis-commons/linkis-common/src/test/resources/linkis.properties similarity index 86% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/resources/conf/user.properties rename to linkis-commons/linkis-common/src/test/resources/linkis.properties index faa650fc5c..230ad85a4e 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/resources/conf/user.properties +++ b/linkis-commons/linkis-common/src/test/resources/linkis.properties @@ -12,8 +12,5 @@ # See the License for the specific language governing permissions and # limitations under the License. # -# -user.props=xxxxxxxxxxxxx -conf.prop.integer=9 -conf.prop.string=str -wds.linkis.client.param.conf.spark.executor.memory=11111G \ No newline at end of file + +linkis.jobhistory.error.msg.tip=properties支持中文 \ No newline at end of file diff --git a/linkis-commons/linkis-common/src/test/scala/org/apache/linkis/common/utils/VariableUtilsTest.scala b/linkis-commons/linkis-common/src/test/scala/org/apache/linkis/common/utils/VariableUtilsTest.scala index c0d4ad1d61..e7a105497c 100644 --- a/linkis-commons/linkis-common/src/test/scala/org/apache/linkis/common/utils/VariableUtilsTest.scala +++ b/linkis-commons/linkis-common/src/test/scala/org/apache/linkis/common/utils/VariableUtilsTest.scala @@ -22,6 +22,8 @@ import org.apache.linkis.common.variable.DateTypeUtils.{getCurHour, getToday} import java.util +import scala.collection.mutable + import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Test @@ -63,4 +65,19 @@ class VariableUtilsTest { assertEquals(VariableUtils.replace(sql, "sql", varMap), resSql) } + @Test + def testGetCustomVar: Unit = { + var scalaCode = "" + + "-------@set globalpara=60--------\n" + + "--@set globalpara2=66\n" + + "select ${globalpara} as globalpara,\n" + + "-- ${globalpara1} as globalpara1, \n" + + "${globalpara2} as globalpara2;\n" + var pythonCode = "" + + val nameAndValue: mutable.Map[String, String] = + VariableUtils.getCustomVar(scalaCode, CodeAndRunTypeUtils.LANGUAGE_TYPE_SQL); + assertEquals(nameAndValue.size, 2) + } + } diff --git a/linkis-commons/linkis-httpclient/src/main/scala/org/apache/linkis/httpclient/AbstractHttpClient.scala b/linkis-commons/linkis-httpclient/src/main/scala/org/apache/linkis/httpclient/AbstractHttpClient.scala index 7b443eb920..6bc8485ade 100644 --- a/linkis-commons/linkis-httpclient/src/main/scala/org/apache/linkis/httpclient/AbstractHttpClient.scala +++ b/linkis-commons/linkis-httpclient/src/main/scala/org/apache/linkis/httpclient/AbstractHttpClient.scala @@ -142,23 +142,31 @@ abstract class AbstractHttpClient(clientConfig: ClientConfig, clientName: String val req = prepareReq(action) val startTime = System.currentTimeMillis val response = executeRequest(req, Some(waitTime).filter(_ > 0)) + val taken = System.currentTimeMillis - startTime + attempts.add(taken) + val costTime = ByteTimeUtils.msDurationToString(taken) + logger.info( + s"invoke ${req.getURI} get status ${response.getStatusLine.getStatusCode} taken: ${costTime}." + ) if (response.getStatusLine.getStatusCode == 401) { tryLogin(action, getRequestUrl(action), true) - logger.info("The user is not logged in, please log in first, you can set a retry") val msg = Utils.tryCatch(EntityUtils.toString(response.getEntity)) { t => logger.warn("failed to parse entity", t) "" } IOUtils.closeQuietly(response) - throw new HttpClientRetryException( - "The user is not logged in, please log in first, you can set a retry, message: " + msg - ) + if (attempts.size() <= 1) { + logger.info("The user is not logged in, default retry once") + addAttempt() + } else { + logger.info("The user is not logged in, you can set a retry") + throw new HttpClientRetryException( + "The user is not logged in, please log in first, you can set a retry, message: " + msg + ) + } + } else { + response } - val taken = System.currentTimeMillis - startTime - attempts.add(taken) - val costTime = ByteTimeUtils.msDurationToString(taken) - logger.info(s"invoke ${req.getURI} taken: ${costTime}.") - response } val response = diff --git a/linkis-commons/linkis-httpclient/src/main/scala/org/apache/linkis/httpclient/config/ClientConfigBuilder.scala b/linkis-commons/linkis-httpclient/src/main/scala/org/apache/linkis/httpclient/config/ClientConfigBuilder.scala index c3e5afba30..b1fc579f3c 100644 --- a/linkis-commons/linkis-httpclient/src/main/scala/org/apache/linkis/httpclient/config/ClientConfigBuilder.scala +++ b/linkis-commons/linkis-httpclient/src/main/scala/org/apache/linkis/httpclient/config/ClientConfigBuilder.scala @@ -17,6 +17,7 @@ package org.apache.linkis.httpclient.config +import org.apache.linkis.common.exception.LinkisRetryException import org.apache.linkis.common.utils.{DefaultRetryHandler, RetryHandler} import org.apache.linkis.httpclient.authentication.AuthenticationStrategy import org.apache.linkis.httpclient.loadbalancer.LoadBalancerStrategy @@ -38,7 +39,12 @@ class ClientConfigBuilder protected () { protected var readTimeout: Long = _ protected var maxConnection: Int = _ protected var retryEnabled: Boolean = true - protected var retryHandler: RetryHandler = new DefaultRetryHandler + + protected var retryHandler: RetryHandler = { + val retryHandler = new DefaultRetryHandler + retryHandler.addRetryException(classOf[LinkisRetryException]) + retryHandler + } def addServerUrl(serverUrl: String): this.type = { this.serverUrl = serverUrl diff --git a/linkis-commons/linkis-module/pom.xml b/linkis-commons/linkis-module/pom.xml index b70331e0d1..cbaf0295fd 100644 --- a/linkis-commons/linkis-module/pom.xml +++ b/linkis-commons/linkis-module/pom.xml @@ -294,6 +294,7 @@ org.springframework.security spring-security-crypto + 5.7.5 diff --git a/linkis-commons/linkis-module/src/main/scala/org/apache/linkis/server/security/SecurityFilter.scala b/linkis-commons/linkis-module/src/main/scala/org/apache/linkis/server/security/SecurityFilter.scala index 411bda4820..5cc796d23e 100644 --- a/linkis-commons/linkis-module/src/main/scala/org/apache/linkis/server/security/SecurityFilter.scala +++ b/linkis-commons/linkis-module/src/main/scala/org/apache/linkis/server/security/SecurityFilter.scala @@ -199,7 +199,11 @@ object SecurityFilter { def getLoginUsername(req: HttpServletRequest): String = { if (Configuration.IS_TEST_MODE.getValue) { - ServerConfiguration.BDP_TEST_USER.getValue; + val testUser = ServerConfiguration.BDP_TEST_USER.getValue + if (StringUtils.isBlank(testUser)) { + throw new IllegalUserTicketException("Need to set test user when enable test module") + } + testUser } else { getLoginUser(req).getOrElse( throw new IllegalUserTicketException(ILLEGAL_USER_TOKEN.getErrorDesc) diff --git a/linkis-commons/linkis-protocol/src/main/java/org/apache/linkis/protocol/constants/TaskConstant.java b/linkis-commons/linkis-protocol/src/main/java/org/apache/linkis/protocol/constants/TaskConstant.java index 42661f8283..6eb97c84d9 100644 --- a/linkis-commons/linkis-protocol/src/main/java/org/apache/linkis/protocol/constants/TaskConstant.java +++ b/linkis-commons/linkis-protocol/src/main/java/org/apache/linkis/protocol/constants/TaskConstant.java @@ -69,6 +69,7 @@ public interface TaskConstant { String TICKET_ID = "ticketId"; String ENGINE_CONN_TASK_ID = "engineConnTaskId"; String ENGINE_CONN_SUBMIT_TIME = "engineConnSubmitTime"; + String DEBUG_ENBALE = "debug.enable"; String PARAMS_DATA_SOURCE = "dataSources"; diff --git a/linkis-commons/linkis-protocol/src/main/scala/org/apache/linkis/protocol/callback/LogCallbackProtocol.scala b/linkis-commons/linkis-protocol/src/main/scala/org/apache/linkis/protocol/callback/LogCallbackProtocol.scala index 91b0e01727..0109472a90 100644 --- a/linkis-commons/linkis-protocol/src/main/scala/org/apache/linkis/protocol/callback/LogCallbackProtocol.scala +++ b/linkis-commons/linkis-protocol/src/main/scala/org/apache/linkis/protocol/callback/LogCallbackProtocol.scala @@ -19,9 +19,6 @@ package org.apache.linkis.protocol.callback import org.apache.linkis.protocol.message.RequestProtocol -// TODO: log type -case class LogCallbackProtocol(nodeId: String, logs: Array[String]) extends RequestProtocol - case class YarnAPPIdCallbackProtocol(nodeId: String, applicationId: String) extends RequestProtocol case class YarnInfoCallbackProtocol(nodeId: String, uri: String) extends RequestProtocol diff --git a/linkis-commons/linkis-protocol/src/main/scala/org/apache/linkis/protocol/utils/TaskUtils.scala b/linkis-commons/linkis-protocol/src/main/scala/org/apache/linkis/protocol/utils/TaskUtils.scala index 3b94bbdc14..9b2be16ef7 100644 --- a/linkis-commons/linkis-protocol/src/main/scala/org/apache/linkis/protocol/utils/TaskUtils.scala +++ b/linkis-commons/linkis-protocol/src/main/scala/org/apache/linkis/protocol/utils/TaskUtils.scala @@ -97,4 +97,15 @@ object TaskUtils { def addLabelsMap(params: util.Map[String, AnyRef], labels: util.Map[String, AnyRef]): Unit = addMap(params, labels, TaskConstant.LABELS) + def isWithDebugInfo(params: util.Map[String, AnyRef]): Boolean = { + val debug = getConfigurationMap(params, TaskConstant.PARAMS_CONFIGURATION_STARTUP).get( + TaskConstant.DEBUG_ENBALE + ) + if (debug != null && "true".equals(debug.toString)) { + true + } else { + false + } + } + } diff --git a/linkis-commons/linkis-rpc/pom.xml b/linkis-commons/linkis-rpc/pom.xml index 3a51a0bfcd..e916107f50 100644 --- a/linkis-commons/linkis-rpc/pom.xml +++ b/linkis-commons/linkis-rpc/pom.xml @@ -56,15 +56,7 @@ com.fasterxml.jackson.core - jackson-annotations - - - com.fasterxml.jackson.core - jackson-core - - - com.fasterxml.jackson.core - jackson-databind + * com.google.code.findbugs @@ -78,8 +70,17 @@ org.springframework.cloud spring-cloud-commons + + commons-fileupload + commons-fileupload + + + commons-fileupload + commons-fileupload + ${commons-fileupload.version} + org.springframework.cloud spring-cloud-commons diff --git a/linkis-commons/linkis-rpc/src/main/scala/org/apache/linkis/rpc/conf/RPCConfiguration.scala b/linkis-commons/linkis-rpc/src/main/scala/org/apache/linkis/rpc/conf/RPCConfiguration.scala index 32f12da273..bbe2d4acd3 100644 --- a/linkis-commons/linkis-rpc/src/main/scala/org/apache/linkis/rpc/conf/RPCConfiguration.scala +++ b/linkis-commons/linkis-rpc/src/main/scala/org/apache/linkis/rpc/conf/RPCConfiguration.scala @@ -65,14 +65,6 @@ object RPCConfiguration { "cs,contextservice,data-source-manager,metadataQuery,metadatamanager,query,jobhistory,application,configuration,filesystem,udf,variable,microservice,errorcode,bml,datasource,basedata-manager" ).getValue.split(",") - val METADATAQUERY_SERVICE_APPLICATION_NAME: CommonVars[String] = - CommonVars("wds.linkis.gateway.conf.publicservice.name", "linkis-ps-metadataquery") - - val METADATAQUERY_SERVICE_LIST: Array[String] = CommonVars( - "wds.linkis.gateway.conf.metadataquery.list", - "metadatamanager,metadataquery" - ).getValue.split(",") - val LINKIS_MANAGER_SERVICE_NAME: CommonVars[String] = CommonVars("wds.linkis.gateway.conf.linkismanager.name", "linkis-cg-linkismanager") @@ -80,6 +72,16 @@ object RPCConfiguration { CommonVars("wds.linkis.gateway.conf.linkismanager.list", "linkisManager,engineplugin").getValue .split(",") + val LINKIS_DATASOURCE_SERVICE_NAME: CommonVars[String] = + CommonVars("linkis.gateway.conf.linkisdatasource.name", "linkis-ps-datasource") + + val LINKIS_DATASOURCE_SERVICE_LIST: Array[String] = + CommonVars( + "linkis.gateway.conf.linkisdatasource.list", + "data-source-manager,metadataquery,datasource,metadataQuery,metadatamanager" + ).getValue + .split(",") + val BDP_RPC_INSTANCE_ALIAS_SERVICE_REFRESH_INTERVAL: CommonVars[TimeType] = CommonVars("wds.linkis.rpc.instancealias.refresh.interval", new TimeType("3s")) diff --git a/linkis-commons/linkis-rpc/src/test/scala/org/apache/linkis/rpc/conf/RPCConfigurationTest.scala b/linkis-commons/linkis-rpc/src/test/scala/org/apache/linkis/rpc/conf/RPCConfigurationTest.scala index 5aaadd133c..ed1d39ce8c 100644 --- a/linkis-commons/linkis-rpc/src/test/scala/org/apache/linkis/rpc/conf/RPCConfigurationTest.scala +++ b/linkis-commons/linkis-rpc/src/test/scala/org/apache/linkis/rpc/conf/RPCConfigurationTest.scala @@ -37,7 +37,6 @@ class RPCConfigurationTest { val enablepublicservice = RPCConfiguration.ENABLE_PUBLIC_SERVICE.getValue val publicserviceapplicationname = RPCConfiguration.PUBLIC_SERVICE_APPLICATION_NAME.getValue val publicservicelist = RPCConfiguration.PUBLIC_SERVICE_LIST - val metadataqueryservicelist = RPCConfiguration.METADATAQUERY_SERVICE_LIST Assertions.assertTrue(25 == bdprpcbroadcastthreadsize.intValue()) Assertions.assertTrue(400 == bdprpcreceiverasynconsumerthreadmax.intValue()) @@ -48,7 +47,6 @@ class RPCConfigurationTest { Assertions.assertTrue(enablepublicservice) Assertions.assertEquals("linkis-ps-publicservice", publicserviceapplicationname) Assertions.assertTrue(publicservicelist.size > 0) - Assertions.assertTrue(metadataqueryservicelist.size > 0) } diff --git a/linkis-commons/linkis-scheduler/src/main/scala/org/apache/linkis/scheduler/queue/Consumer.scala b/linkis-commons/linkis-scheduler/src/main/scala/org/apache/linkis/scheduler/queue/Consumer.scala index 50dce2ca12..165a274362 100644 --- a/linkis-commons/linkis-scheduler/src/main/scala/org/apache/linkis/scheduler/queue/Consumer.scala +++ b/linkis-commons/linkis-scheduler/src/main/scala/org/apache/linkis/scheduler/queue/Consumer.scala @@ -41,7 +41,6 @@ abstract class Consumer(schedulerContext: SchedulerContext, executeService: Exec def start(): Unit def shutdown(): Unit = { - logger.info(s"$toString is ready to stop!") terminate = true logger.info(s"$toString stopped!") } diff --git a/linkis-commons/linkis-scheduler/src/main/scala/org/apache/linkis/scheduler/queue/GroupFactory.scala b/linkis-commons/linkis-scheduler/src/main/scala/org/apache/linkis/scheduler/queue/GroupFactory.scala index f3471b07dd..be1716f238 100644 --- a/linkis-commons/linkis-scheduler/src/main/scala/org/apache/linkis/scheduler/queue/GroupFactory.scala +++ b/linkis-commons/linkis-scheduler/src/main/scala/org/apache/linkis/scheduler/queue/GroupFactory.scala @@ -19,6 +19,11 @@ package org.apache.linkis.scheduler.queue abstract class GroupFactory { + /** + * Create a Group and set the concurrency limit of the group + * @param event + * @return + */ def getOrCreateGroup(event: SchedulerEvent): Group def getGroup(groupName: String): Group diff --git a/linkis-commons/linkis-scheduler/src/main/scala/org/apache/linkis/scheduler/queue/LoopArrayQueue.scala b/linkis-commons/linkis-scheduler/src/main/scala/org/apache/linkis/scheduler/queue/LoopArrayQueue.scala index b0bbfd3c2b..8bea7e52b1 100644 --- a/linkis-commons/linkis-scheduler/src/main/scala/org/apache/linkis/scheduler/queue/LoopArrayQueue.scala +++ b/linkis-commons/linkis-scheduler/src/main/scala/org/apache/linkis/scheduler/queue/LoopArrayQueue.scala @@ -40,7 +40,12 @@ class LoopArrayQueue(var group: Group) extends ConsumeQueue with Logging { override def getWaitingEvents: Array[SchedulerEvent] = { eventQueue synchronized { - toIndexedSeq.filter(x => x.getState.equals(SchedulerEventState.Inited)).toArray + toIndexedSeq + .filter(x => + x.getState.equals(SchedulerEventState.Inited) || x.getState + .equals(SchedulerEventState.Scheduled) + ) + .toArray } } diff --git a/linkis-commons/linkis-scheduler/src/main/scala/org/apache/linkis/scheduler/queue/fifoqueue/FIFOUserConsumer.scala b/linkis-commons/linkis-scheduler/src/main/scala/org/apache/linkis/scheduler/queue/fifoqueue/FIFOUserConsumer.scala index 2a40c2517b..d541d8a2eb 100644 --- a/linkis-commons/linkis-scheduler/src/main/scala/org/apache/linkis/scheduler/queue/fifoqueue/FIFOUserConsumer.scala +++ b/linkis-commons/linkis-scheduler/src/main/scala/org/apache/linkis/scheduler/queue/fifoqueue/FIFOUserConsumer.scala @@ -73,6 +73,8 @@ class FIFOUserConsumer( override def getRunningEvents: Array[SchedulerEvent] = getEvents(e => e.isRunning || e.isWaitForRetry) + protected def getSchedulerContext: SchedulerContext = schedulerContext + private def getEvents(op: SchedulerEvent => Boolean): Array[SchedulerEvent] = { val result = ArrayBuffer[SchedulerEvent]() runningJobs.filter(_ != null).filter(x => op(x)).foreach(result += _) @@ -82,16 +84,28 @@ class FIFOUserConsumer( override def run(): Unit = { Thread.currentThread().setName(s"${toString}Thread") logger.info(s"$toString thread started!") - while (!terminate) { - Utils.tryAndError(loop()) - Utils.tryAndError(Thread.sleep(10)) + while (!terminate) Utils.tryAndError { + loop() + Thread.sleep(10) } logger.info(s"$toString thread stopped!") } protected def askExecutorGap(): Unit = {} + /** + * Task scheduling interception is used to judge the rules of task operation, and to judge other + * task rules based on Group. For example, Entrance makes Creator-level task judgment. + */ + protected def runScheduleIntercept(): Boolean = { + true + } + protected def loop(): Unit = { + if (!runScheduleIntercept()) { + Utils.tryQuietly(Thread.sleep(1000)) + return + } var isRetryJob = false def getWaitForRetryEvent: Option[SchedulerEvent] = { val waitForRetryJobs = runningJobs.filter(job => job != null && job.isJobCanRetry) @@ -110,7 +124,7 @@ class FIFOUserConsumer( if (event.isEmpty) { val completedNums = runningJobs.filter(job => job == null || job.isCompleted) if (completedNums.length < 1) { - Utils.tryQuietly(Thread.sleep(1000)) // TODO 还可以优化，通过实现JobListener进行优化 + Utils.tryQuietly(Thread.sleep(1000)) return } while (event.isEmpty) { @@ -119,7 +133,12 @@ class FIFOUserConsumer( if ( takeEvent.exists(e => Utils.tryCatch(e.turnToScheduled()) { t => - takeEvent.get.asInstanceOf[Job].onFailure("Job状态翻转为Scheduled失败！", t) + takeEvent.get + .asInstanceOf[Job] + .onFailure( + "Failed to change the job status to Scheduled(Job状态翻转为Scheduled失败)", + t + ) false } ) @@ -174,7 +193,7 @@ class FIFOUserConsumer( ) ) case error: Throwable => - job.onFailure("请求引擎失败，可能是由于后台进程错误!请联系管理员", error) + job.onFailure("Failed to request EngineConn", error) if (job.isWaitForRetry) { logger.warn(s"Ask executor for Job $job failed, wait for the next retry!", error) if (!isRetryJob) putToRunningJobs(job) @@ -190,6 +209,20 @@ class FIFOUserConsumer( override def shutdown(): Unit = { future.cancel(true) + val waitEvents = queue.getWaitingEvents + if (waitEvents.nonEmpty) { + waitEvents.foreach { + case job: Job => + job.onFailure("Your job will be marked as canceled because the consumer be killed", null) + case _ => + } + } + + this.runningJobs.foreach { job => + if (job != null && !job.isCompleted) { + job.onFailure("Your job will be marked as canceled because the consumer be killed", null) + } + } super.shutdown() } diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/excel/XlsUtils.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/excel/XlsUtils.java index f3b8f44865..68469cb520 100644 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/excel/XlsUtils.java +++ b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/excel/XlsUtils.java @@ -61,6 +61,7 @@ public static String excelToCsv( throws Exception { String hdfsPath = "/tmp/" + StorageUtils.getJvmUser() + "/" + System.currentTimeMillis() + ".csv"; + LOG.info("The excel to csv with hdfs path:" + hdfsPath); ExcelXlsReader xlsReader = new ExcelXlsReader(); RowToCsvDeal rowToCsvDeal = new RowToCsvDeal(); OutputStream out = null; diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/exception/StorageErrorCode.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/exception/StorageErrorCode.java index 3c82ceb523..8c0d34488a 100644 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/exception/StorageErrorCode.java +++ b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/exception/StorageErrorCode.java @@ -20,7 +20,9 @@ public enum StorageErrorCode { /** */ - FS_NOT_INIT(53001, "please init first(请先初始化)"); + FS_NOT_INIT(53001, "please init first"), + INCONSISTENT_DATA(53001, "Inconsistent row data read,read %s,need rowLen %s"), + FS_OOM(53002, "OOM occurred while reading the file"); StorageErrorCode(int errorCode, String message) { this.code = errorCode; diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/fs/FileSystem.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/fs/FileSystem.java index c19213a9cc..4bb7cb33b4 100644 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/fs/FileSystem.java +++ b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/fs/FileSystem.java @@ -24,8 +24,13 @@ import java.io.File; import java.io.IOException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + public abstract class FileSystem implements Fs { + private static final Logger LOG = LoggerFactory.getLogger(FileSystem.class); + protected String user; private String defaultFilePerm = "rwxr-----"; // 740 private String defaultFolderPerm = "rwxr-x---"; // 750 @@ -94,6 +99,7 @@ protected FsPath getParentPath(String path) { } else { parentPath = path.substring(0, path.lastIndexOf("/")); } + LOG.info("Get parent path:" + parentPath); return new FsPath(parentPath); } diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/fs/impl/LocalFileSystem.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/fs/impl/LocalFileSystem.java index ce2ee43b7e..2df547f10e 100644 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/fs/impl/LocalFileSystem.java +++ b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/fs/impl/LocalFileSystem.java @@ -75,12 +75,14 @@ public String listRoot() throws IOException { @Override public long getTotalSpace(FsPath dest) throws IOException { String path = dest.getPath(); + LOG.info("Get total space with path:" + path); return new File(path).getTotalSpace(); } @Override public long getFreeSpace(FsPath dest) throws IOException { String path = dest.getPath(); + LOG.info("Get free space with path:" + path); return new File(path).getFreeSpace(); } @@ -117,6 +119,7 @@ public boolean setOwner(FsPath dest, String user, String group) throws IOExcepti @Override public boolean setOwner(FsPath dest, String user) throws IOException { + LOG.info("Set owner with path:" + dest.getPath() + "and user:" + user); if (!StorageUtils.isIOProxy()) { LOG.info("io not proxy, setOwner skip"); return true; @@ -133,6 +136,7 @@ public boolean setOwner(FsPath dest, String user) throws IOException { @Override public boolean setGroup(FsPath dest, String group) throws IOException { + LOG.info("Set group with path:" + dest.getPath() + "and group:" + user); if (!StorageUtils.isIOProxy()) { LOG.info("io not proxy, setGroup skip"); return true; @@ -155,6 +159,7 @@ public boolean mkdir(FsPath dest) throws IOException { @Override public boolean mkdirs(FsPath dest) throws IOException { String path = dest.getPath(); + LOG.info("Try to mkdirs with path:" + path); File file = new File(path); // Create parent directories one by one and set their permissions to rwxrwxrwx. Stack dirsToMake = new Stack(); @@ -182,6 +187,7 @@ public boolean mkdirs(FsPath dest) throws IOException { } public boolean canMkdir(FsPath destParentDir) throws IOException { + LOG.info("Try to check if the directory can be created with path:" + destParentDir.getPath()); if (!StorageUtils.isIOProxy()) { LOG.debug("io not proxy, not check owner, just check if have write permission "); return this.canWrite(destParentDir); @@ -203,6 +209,7 @@ public boolean canMkdir(FsPath destParentDir) throws IOException { @Override public boolean copy(String origin, String dest) throws IOException { File file = new File(dest); + LOG.info("Try to copy file from:" + origin + " to dest:" + dest); if (!isOwner(file.getParent())) { throw new IOException("you have on permission to create file " + dest); } @@ -225,6 +232,7 @@ public boolean copy(String origin, String dest) throws IOException { @Override public boolean setPermission(FsPath dest, String permission) throws IOException { + LOG.info("Try to set permission dest with path:" + dest.getPath()); if (!StorageUtils.isIOProxy()) { LOG.info("io not proxy, setPermission as parent."); try { @@ -251,6 +259,7 @@ public boolean setPermission(FsPath dest, String permission) throws IOException public FsPathListWithError listPathWithError(FsPath path) throws IOException { File file = new File(path.getPath()); File[] files = file.listFiles(); + LOG.info("Try to list path:" + path.getPath() + " with error msg"); if (files != null) { List rtn = new ArrayList(); String message = ""; @@ -294,6 +303,7 @@ public void init(Map properties) throws IOException { String groupInfo; try { groupInfo = Utils.exec(new String[] {"id", user}); + LOG.info("Get groupinfo:" + groupInfo + " with shell command: id " + user); } catch (RuntimeException e) { group = user; return; @@ -322,7 +332,7 @@ public FsPath get(String dest) throws IOException { } else { fsPath = new FsPath(dest); } - + LOG.info("Try to get FsPath with path:" + fsPath.getPath()); PosixFileAttributes attr = null; try { attr = Files.readAttributes(Paths.get(fsPath.getPath()), PosixFileAttributes.class); @@ -365,7 +375,7 @@ public OutputStream write(FsPath dest, boolean overwrite) throws IOException { @Override public boolean create(String dest) throws IOException { - + LOG.info("try to create file with path:" + dest); File file = new File(dest); if (!isOwner(file.getParent())) { throw new IOException("you have on permission to create file " + dest); @@ -391,6 +401,7 @@ public boolean create(String dest) throws IOException { public List list(FsPath path) throws IOException { File file = new File(path.getPath()); File[] files = file.listFiles(); + LOG.info("Try to get file list with path:" + path.getPath()); if (files != null) { List rtn = new ArrayList(); for (File f : files) { diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/conf/LinkisStorageConf.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/conf/LinkisStorageConf.scala index bf03c78754..a76fe82fcb 100644 --- a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/conf/LinkisStorageConf.scala +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/conf/LinkisStorageConf.scala @@ -39,7 +39,7 @@ object LinkisStorageConf { val FILE_TYPE = CommonVars( "wds.linkis.storage.file.type", - "dolphin,sql,scala,py,hql,python,out,log,text,sh,jdbc,ngql,psql,fql,tsql" + "dolphin,sql,scala,py,hql,python,out,log,text,sh,jdbc,ngql,psql,fql,tsql,txt" ).getValue private var fileTypeArr: Array[String] = null @@ -60,4 +60,5 @@ object LinkisStorageConf { fileTypeArr } + val LINKIS_RESULT_ENABLE_NULL = CommonVars("linkis.resultset.enable.null.replace", false).getValue } diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/csv/StorageCSVWriter.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/csv/StorageCSVWriter.scala index f9b811b6a6..63dce31ca4 100644 --- a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/csv/StorageCSVWriter.scala +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/csv/StorageCSVWriter.scala @@ -36,6 +36,8 @@ class StorageCSVWriter( with Logging { private val delimiter = separator match { + // Compatible with possible missing escape characters + case "t" => '\t' case separ if StringUtils.isNotEmpty(separ) => separ case _ => '\t' } @@ -50,14 +52,21 @@ class StorageCSVWriter( private def compact(row: Array[String]): String = { val quotationMarks: String = "\"" + val dealNewlineSymbolMarks: String = "\n" + def decorateValue(v: String): String = { if (StringUtils.isBlank(v)) v else { + var res = v if (quoteRetouchEnable) { - s"$quotationMarks${v.replaceAll(quotationMarks, "")}$quotationMarks" - } else v + res = s"$quotationMarks${v.replaceAll(quotationMarks, "")}$quotationMarks" + } + res = res.replaceAll(dealNewlineSymbolMarks, " ") + logger.debug("decorateValue with input:" + v + " output:" + res) + res } } + if (logger.isDebugEnabled()) { logger.debug("delimiter:" + delimiter.toString) } diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/domain/DataType.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/domain/DataType.scala index 55c82abd38..69182fc6bd 100644 --- a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/domain/DataType.scala +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/domain/DataType.scala @@ -18,16 +18,16 @@ package org.apache.linkis.storage.domain import org.apache.linkis.common.utils.{Logging, Utils} +import org.apache.linkis.storage.conf.LinkisStorageConf import java.math.{BigDecimal => JavaBigDecimal} import java.sql.{Date, Timestamp} object DataType extends Logging { - val NULL_VALUE = "NULL" val LOWCASE_NULL_VALUE = "null" - // TODO Change to fine-grained regular expressions(改为精细化正则表达式) - val DECIMAL_REGEX = "^decimal\\(\\d*\\,\\d*\\)".r.unanchored + + val DECIMAL_REGEX = "^decimal\\(\\s*\\d*\\s*,\\s*\\d*\\s*\\)".r.unanchored val SHORT_REGEX = "^short.*".r.unanchored val INT_REGEX = "^int.*".r.unanchored @@ -70,39 +70,53 @@ object DataType extends Logging { case _ => StringType } - def toValue(dataType: DataType, value: String): Any = Utils.tryCatch(dataType match { - case NullType => null - case StringType | CharType | VarcharType | StructType | ListType | ArrayType | MapType => - value - case BooleanType => if (isNumberNull(value)) null else value.toBoolean - case ShortIntType => if (isNumberNull(value)) null else value.toShort - case IntType => if (isNumberNull(value)) null else value.toInt - case LongType | BigIntType => if (isNumberNull(value)) null else value.toLong - case FloatType => if (isNumberNull(value)) null else value.toFloat - case DoubleType => if (isNumberNull(value)) null else value.toDouble - case DecimalType => if (isNumberNull(value)) null else new JavaBigDecimal(value) - case DateType => if (isNumberNull(value)) null else Date.valueOf(value) - case TimestampType => - if (isNumberNull(value)) null else Timestamp.valueOf(value).toString.stripSuffix(".0") - case BinaryType => if (isNull(value)) null else value.getBytes() - case _ => value - }) { t => - logger.debug(s"Failed to $value switch to dataType:", t) - value + def toValue(dataType: DataType, value: String): Any = { + var newValue: String = value + if (isLinkisNull(value)) { + if (!LinkisStorageConf.LINKIS_RESULT_ENABLE_NULL) { + return null + } else { + newValue = Dolphin.NULL + } + } + Utils.tryCatch(dataType match { + case NullType => null + case StringType | CharType | VarcharType | StructType | ListType | ArrayType | MapType => + newValue + case BooleanType => if (isNumberNull(newValue)) null else newValue.toBoolean + case ShortIntType => if (isNumberNull(newValue)) null else newValue.toShort + case IntType => if (isNumberNull(newValue)) null else newValue.toInt + case LongType | BigIntType => if (isNumberNull(newValue)) null else newValue.toLong + case FloatType => if (isNumberNull(newValue)) null else newValue.toFloat + case DoubleType => if (isNumberNull(newValue)) null else newValue.toDouble + case DecimalType => if (isNumberNull(newValue)) null else new JavaBigDecimal(newValue) + case DateType => if (isNumberNull(newValue)) null else Date.valueOf(newValue) + case TimestampType => + if (isNumberNull(newValue)) null else Timestamp.valueOf(newValue).toString.stripSuffix(".0") + case BinaryType => if (isNull(newValue)) null else newValue.getBytes() + case _ => newValue + }) { t => + logger.debug(s"Failed to $newValue switch to dataType:", t) + newValue + } + } + + def isLinkisNull(value: String): Boolean = { + if (value == null || value == Dolphin.LINKIS_NULL) true else false } def isNull(value: String): Boolean = - if (value == null || value == NULL_VALUE || value.trim == "") true else false + if (value == null || value == Dolphin.NULL || value.trim == "") true else false def isNumberNull(value: String): Boolean = - if (null == value || NULL_VALUE.equalsIgnoreCase(value) || value.trim == "") { + if (null == value || Dolphin.NULL.equalsIgnoreCase(value) || value.trim == "") { true } else { false } def valueToString(value: Any): String = { - if (null == value) return LOWCASE_NULL_VALUE + if (null == value) return null value match { case javaDecimal: JavaBigDecimal => javaDecimal.toPlainString diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/domain/Dolphin.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/domain/Dolphin.scala index 378c2c2ecb..8e7e7d6131 100644 --- a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/domain/Dolphin.scala +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/domain/Dolphin.scala @@ -39,7 +39,10 @@ object Dolphin extends Logging { val COL_SPLIT_LEN = COL_SPLIT_BYTES.length val NULL = "NULL" - val NULL_BYTES = "NULL".getBytes("utf-8") + val NULL_BYTES = NULL.getBytes("utf-8") + + val LINKIS_NULL = "LINKIS_NULL" + val LINKIS_NULL_BYTES = LINKIS_NULL.getBytes("utf-8") val INT_LEN = 10 @@ -59,6 +62,14 @@ object Dolphin extends Logging { def getString(bytes: Array[Byte], start: Int, len: Int): String = new String(bytes, start, len, Dolphin.CHAR_SET) + def toStringValue(value: String): String = { + if (LINKIS_NULL.equals(value)) { + NULL + } else { + value + } + } + /** * Read an integer value that converts the array to a byte of length 10 bytes * 读取整数值，该值为将数组转换为10字节长度的byte diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/DefaultResultSetFactory.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/DefaultResultSetFactory.scala index 38973ae1ab..d4836731db 100644 --- a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/DefaultResultSetFactory.scala +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/DefaultResultSetFactory.scala @@ -108,7 +108,7 @@ class DefaultResultSetFactory extends ResultSetFactory with Logging { if (StringUtils.isEmpty(resultSetType)) { throw new StorageWarnException( THE_FILE_IS_EMPTY.getErrorCode, - s"The file (${fsPath.getPath}) is empty(文件(${fsPath.getPath}) 为空)" + MessageFormat.format(THE_FILE_IS_EMPTY.getErrorDesc, fsPath.getPath) ) } Utils.tryQuietly(inputStream.close()) diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/ResultSetReader.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/ResultSetReader.scala index 663e379b5b..e61cf36b3d 100644 --- a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/ResultSetReader.scala +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/ResultSetReader.scala @@ -19,6 +19,7 @@ package org.apache.linkis.storage.resultset import org.apache.linkis.common.io.{FsPath, MetaData, Record} import org.apache.linkis.common.io.resultset.{ResultSet, ResultSetReader} +import org.apache.linkis.common.utils.Logging import org.apache.linkis.storage.FSFactory import org.apache.linkis.storage.errorcode.LinkisStorageErrorCodeSummary.TABLE_ARE_NOT_SUPPORTED import org.apache.linkis.storage.exception.StorageErrorException @@ -26,7 +27,7 @@ import org.apache.linkis.storage.resultset.table.{TableMetaData, TableRecord, Ta import java.io.InputStream -object ResultSetReader { +object ResultSetReader extends Logging { def getResultSetReader[K <: MetaData, V <: Record]( resultSet: ResultSet[K, V], @@ -83,6 +84,7 @@ object ResultSetReader { ) } val fs = FSFactory.getFs(resPath) + logger.info("Try to init Fs with path:" + resPath.getPath) fs.init(null) ResultSetReader.getResultSetReader(resultSet.asInstanceOf[TableResultSet], fs.read(resPath)) } diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/StorageResultSet.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/StorageResultSet.scala index fc303fbb5c..7b3aca62d9 100644 --- a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/StorageResultSet.scala +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/StorageResultSet.scala @@ -34,6 +34,7 @@ abstract class StorageResultSet[K <: MetaData, V <: Record] extends ResultSet[K, } else { parentDir.toPath + "/" + fileName + Dolphin.DOLPHIN_FILE_SUFFIX } + logger.info(s"Get result set path:${path}") new FsPath(path) } diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/StorageResultSetReader.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/StorageResultSetReader.scala index 3f49faf3ed..80db410746 100644 --- a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/StorageResultSetReader.scala +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/StorageResultSetReader.scala @@ -21,7 +21,11 @@ import org.apache.linkis.common.io.{Fs, MetaData, Record} import org.apache.linkis.common.io.resultset.{ResultSet, ResultSetReader} import org.apache.linkis.common.utils.{Logging, Utils} import org.apache.linkis.storage.domain.Dolphin -import org.apache.linkis.storage.exception.StorageWarnException +import org.apache.linkis.storage.exception.{ + StorageErrorCode, + StorageErrorException, + StorageWarnException +} import org.apache.linkis.storage.utils.StorageUtils import java.io.{ByteArrayInputStream, InputStream, IOException} @@ -43,7 +47,6 @@ class StorageResultSetReader[K <: MetaData, V <: Record]( private var fs: Fs = _ private val READ_CACHE = 1024 - private val bytes = new Array[Byte](READ_CACHE) def this(resultSet: ResultSet[K, V], value: String) = { this(resultSet, new ByteArrayInputStream(value.getBytes(Dolphin.CHAR_SET))) @@ -74,24 +77,27 @@ class StorageResultSetReader[K <: MetaData, V <: Record]( case t: Throwable => throw t } - val rowBuffer = ArrayBuffer[Byte]() - var len = 0 - - // Read the entire line, except for the data of the line length(读取整行，除了行长的数据) - while (rowLen > 0 && len >= 0) { - if (rowLen > READ_CACHE) { - len = StorageUtils.readBytes(inputStream, bytes, READ_CACHE) - } else { - len = StorageUtils.readBytes(inputStream, bytes, rowLen) - } - - if (len > 0) { - rowLen -= len - rowBuffer ++= bytes.slice(0, len) - } + var bytes: Array[Byte] = null + try { + bytes = new Array[Byte](rowLen) + } catch { + case e: OutOfMemoryError => + logger.error("Result set read oom, read size {} Byte", rowLen) + throw new StorageErrorException( + StorageErrorCode.FS_OOM.getCode, + StorageErrorCode.FS_OOM.getMessage, + e + ) + } + val len = StorageUtils.readBytes(inputStream, bytes, rowLen) + if (len != rowLen) { + throw new StorageErrorException( + StorageErrorCode.INCONSISTENT_DATA.getCode, + String.format(StorageErrorCode.INCONSISTENT_DATA.getMessage, len.toString, rowLen.toString) + ) } rowCount = rowCount + 1 - rowBuffer.toArray + bytes } @scala.throws[IOException] diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/StorageResultSetWriter.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/StorageResultSetWriter.scala index 9c7947272c..17106dca67 100644 --- a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/StorageResultSetWriter.scala +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/StorageResultSetWriter.scala @@ -86,6 +86,7 @@ class StorageResultSetWriter[K <: MetaData, V <: Record]( WRITER_LOCK_CREATE.synchronized { if (!fileCreated) { if (storePath != null && outputStream == null) { + logger.info(s"Try to create a new file:${storePath}, with proxy user:${proxyUser}") fs = FSFactory.getFsByProxyUser(storePath, proxyUser) fs.init(null) FileSystemUtils.createNewFile(storePath, proxyUser, true) diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/table/TableRecord.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/table/TableRecord.scala index d8e0560964..64d2b18b9c 100644 --- a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/table/TableRecord.scala +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/table/TableRecord.scala @@ -27,10 +27,4 @@ class TableRecord(val row: Array[Any]) extends ResultRecord { new TableRecord(row) } - def tableRecordToString(nullValue: String = "NULL"): Array[String] = { - row.map { col => - StorageUtils.colToString(col, nullValue) - } - } - } diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/table/TableResultDeserializer.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/table/TableResultDeserializer.scala index 40c4e031f7..32ea4f5e22 100644 --- a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/table/TableResultDeserializer.scala +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/table/TableResultDeserializer.scala @@ -47,13 +47,13 @@ class TableResultDeserializer extends ResultDeserializer[TableMetaData, TableRec val columns = new ArrayBuffer[Column]() for (i <- 0 until (colArray.length, 3)) { var len = colArray(i).toInt - val colName = Dolphin.getString(bytes, index, len) + val colName = Dolphin.toStringValue(Dolphin.getString(bytes, index, len)) index += len len = colArray(i + 1).toInt - val colType = Dolphin.getString(bytes, index, len) + val colType = Dolphin.toStringValue(Dolphin.getString(bytes, index, len)) index += len len = colArray(i + 2).toInt - val colComment = Dolphin.getString(bytes, index, len) + val colComment = Dolphin.toStringValue(Dolphin.getString(bytes, index, len)) index += len columns += Column(colName, colType, colComment) } diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/table/TableResultSerializer.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/table/TableResultSerializer.scala index 94b6cb4c03..5d1738a346 100644 --- a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/table/TableResultSerializer.scala +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/table/TableResultSerializer.scala @@ -19,6 +19,7 @@ package org.apache.linkis.storage.resultset.table import org.apache.linkis.common.io.{MetaData, Record} import org.apache.linkis.common.io.resultset.ResultSerializer +import org.apache.linkis.storage.conf.LinkisStorageConf import org.apache.linkis.storage.domain.Dolphin import scala.collection.mutable.ArrayBuffer @@ -45,14 +46,20 @@ class TableResultSerializer extends ResultSerializer { * @param line */ def lineToBytes(line: Array[Any]): Array[Byte] = { - // Data cache(数据缓存) val dataBytes = ArrayBuffer[Array[Byte]]() - // Column cache(列缓存) val colIndex = ArrayBuffer[Array[Byte]]() var colByteLen = 0 var length = 0 line.foreach { data => - val bytes = if (data == null) Dolphin.NULL_BYTES else Dolphin.getBytes(data) + val bytes = if (data == null) { + if (!LinkisStorageConf.LINKIS_RESULT_ENABLE_NULL) { + Dolphin.LINKIS_NULL_BYTES + } else { + Dolphin.NULL_BYTES + } + } else { + Dolphin.getBytes(data) + } dataBytes += bytes val colBytes = Dolphin.getBytes(bytes.length) colIndex += colBytes += Dolphin.COL_SPLIT_BYTES diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/source/FileSource.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/source/FileSource.scala index 7a9fa4f04c..b6830a13b4 100644 --- a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/source/FileSource.scala +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/source/FileSource.scala @@ -18,6 +18,7 @@ package org.apache.linkis.storage.source import org.apache.linkis.common.io._ +import org.apache.linkis.common.utils.Logging import org.apache.linkis.storage.conf.LinkisStorageConf import org.apache.linkis.storage.errorcode.LinkisStorageErrorCodeSummary.UNSUPPORTED_OPEN_FILE_TYPE import org.apache.linkis.storage.exception.StorageErrorException @@ -55,7 +56,7 @@ trait FileSource extends Closeable { def getFileSplits: Array[FileSplit] } -object FileSource { +object FileSource extends Logging { private val fileType = LinkisStorageConf.getFileTypeArr private val suffixPredicate = (path: String, suffix: String) => path.endsWith(s".$suffix") @@ -124,6 +125,7 @@ object FileSource { } private def createResultSetFileSplit(fsPath: FsPath, fs: Fs): FileSplit = { + logger.info(s"try create result set file split with path:${fsPath.getPath}") val resultset = ResultSetFactory.getInstance.getResultSetByPath(fsPath, fs) val resultsetReader = ResultSetReader.getResultSetReader(resultset, fs.read(fsPath)) new FileSplit(resultsetReader, resultset.resultSetType()) diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/source/ResultsetFileSource.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/source/ResultsetFileSource.scala index 7b387e9f19..adbb596aa2 100644 --- a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/source/ResultsetFileSource.scala +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/source/ResultsetFileSource.scala @@ -17,6 +17,7 @@ package org.apache.linkis.storage.source +import org.apache.linkis.storage.domain.Dolphin import org.apache.linkis.storage.resultset.table.TableRecord import org.apache.linkis.storage.utils.StorageUtils @@ -24,11 +25,27 @@ class ResultsetFileSource(fileSplits: Array[FileSplit]) extends AbstractFileSour shuffle({ case t: TableRecord => - new TableRecord(t.row.map { - case null | "NULL" => getParams.getOrDefault("nullValue", "NULL") - case "" => getParams.getOrDefault("nullValue", "") - case value: Double => StorageUtils.doubleToString(value) - case r => r + new TableRecord(t.row.map { rvalue => + { + rvalue match { + case null | "NULL" => + val nullValue = getParams.getOrDefault("nullValue", "NULL") + if (nullValue.equals(Dolphin.LINKIS_NULL)) { + rvalue + } else { + nullValue + } + case "" => + val nullValue = getParams.getOrDefault("nullValue", "") + if (nullValue.equals(Dolphin.LINKIS_NULL)) { + "" + } else { + nullValue + } + case value: Double => StorageUtils.doubleToString(value) + case _ => rvalue + } + } }) case record => record }) diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/utils/StorageUtils.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/utils/StorageUtils.scala index 90eb319fa0..7ee53a45cf 100644 --- a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/utils/StorageUtils.scala +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/utils/StorageUtils.scala @@ -46,7 +46,11 @@ object StorageUtils extends Logging { nf.setMaximumFractionDigits(StorageConfiguration.DOUBLE_FRACTION_LEN.getValue) def doubleToString(value: Double): String = { - nf.format(value) + if (value.isNaN) { + "NaN" + } else { + nf.format(value) + } } def loadClass[T](classStr: String, op: T => String): Map[String, T] = { @@ -211,19 +215,12 @@ object StorageUtils extends Logging { readLen } - def colToString(col: Any, nullValue: String = "NULL"): String = { - if (null == col) nullValue - else { - col match { - case value: Double => doubleToString(value) - case "NULL" | "" => nullValue - case _ => col.toString - } - } - } - def isIOProxy(): Boolean = { StorageConfiguration.ENABLE_IO_PROXY.getValue } + def isHDFSPath(fsPath: FsPath): Boolean = { + HDFS.equals(fsPath.getFsType) + } + } diff --git a/linkis-commons/linkis-storage/src/test/scala/org/apache/linkis/storage/domain/DataTypeTest.scala b/linkis-commons/linkis-storage/src/test/scala/org/apache/linkis/storage/domain/DataTypeTest.scala index 3bfc35b17c..e0d3d7efb6 100644 --- a/linkis-commons/linkis-storage/src/test/scala/org/apache/linkis/storage/domain/DataTypeTest.scala +++ b/linkis-commons/linkis-storage/src/test/scala/org/apache/linkis/storage/domain/DataTypeTest.scala @@ -25,7 +25,7 @@ class DataTypeTest { @DisplayName("constTest") def constTest(): Unit = { - val nullvalue = DataType.NULL_VALUE + val nullvalue = Dolphin.NULL val lowcasenullvalue = DataType.LOWCASE_NULL_VALUE Assertions.assertEquals("NULL", nullvalue) @@ -60,4 +60,19 @@ class DataTypeTest { } + @Test + @DisplayName("toValueTest") + def toValueTest(): Unit = { + val dateType = DataType.toDataType("double") + val str = DataType.toValue(dateType, "NaN") + Assertions.assertNotNull(str) + } + + @Test + @DisplayName("decimalTest") + def decimalTest(): Unit = { + val dateType = DataType.toDataType("decimal(10, 8)") + Assertions.assertTrue(dateType.typeName.equals("decimal")) + } + } diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/job/JobData.java b/linkis-commons/linkis-storage/src/test/scala/org/apache/linkis/storage/utils/StorageUtilsTest.scala similarity index 72% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/job/JobData.java rename to linkis-commons/linkis-storage/src/test/scala/org/apache/linkis/storage/utils/StorageUtilsTest.scala index 10d55e6541..6ae15782ee 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/job/JobData.java +++ b/linkis-commons/linkis-storage/src/test/scala/org/apache/linkis/storage/utils/StorageUtilsTest.scala @@ -15,20 +15,18 @@ * limitations under the License. */ -package org.apache.linkis.cli.common.entity.job; +package org.apache.linkis.storage.utils -public interface JobData { - JobStatus getJobStatus(); +import org.junit.jupiter.api.{Assertions, DisplayName, Test} - String getJobID(); +class StorageUtilsTest { - String getUser(); + @Test + @DisplayName("doubleToStringTest") + def doubleToStringTest(): Unit = { + val str = StorageUtils.doubleToString(Double.NaN) + Assertions.assertEquals("NaN", str) - String getMessage(); + } - void setMessage(String message); - - Exception getException(); - - void setException(Exception e); } diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/pom.xml b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/pom.xml deleted file mode 100644 index 980141f9a3..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/pom.xml +++ /dev/null @@ -1,87 +0,0 @@ - - - - 4.0.0 - - - org.apache.linkis - linkis-cli - ${revision} - - linkis-cli-application - jar - - - - org.apache.linkis - linkis-cli-common - ${project.version} - - - org.apache.linkis - linkis-cli-core - ${project.version} - - - org.apache.linkis - linkis-computation-client - ${project.version} - - - org.apache.linkis - linkis-gateway-httpclient-support - ${project.version} - - - - - - - - org.apache.maven.plugins - maven-assembly-plugin - false - - false - out - false - false - - /src/main/assembly/distribution.xml - - - - - make-assembly - - single - - package - - - /src/main/assembly/distribution.xml - - - - - - - - - - diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/LinkisClientApplication.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/LinkisClientApplication.java deleted file mode 100644 index 0c6fc1ec73..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/LinkisClientApplication.java +++ /dev/null @@ -1,341 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.application; - -import org.apache.linkis.cli.application.constants.AppConstants; -import org.apache.linkis.cli.application.constants.AppKeys; -import org.apache.linkis.cli.application.data.FinishedData; -import org.apache.linkis.cli.application.data.PreparedData; -import org.apache.linkis.cli.application.data.ProcessedData; -import org.apache.linkis.cli.application.interactor.command.LinkisCmdType; -import org.apache.linkis.cli.application.interactor.command.template.UniversalCmdTemplate; -import org.apache.linkis.cli.application.operator.ujes.LinkisOperatorBuilder; -import org.apache.linkis.cli.application.suite.ExecutionSuite; -import org.apache.linkis.cli.application.suite.ExecutionSuiteFactory; -import org.apache.linkis.cli.application.utils.Utils; -import org.apache.linkis.cli.common.entity.command.CmdTemplate; -import org.apache.linkis.cli.common.entity.command.Params; -import org.apache.linkis.cli.common.entity.execution.Execution; -import org.apache.linkis.cli.common.entity.job.Job; -import org.apache.linkis.cli.common.entity.properties.ClientProperties; -import org.apache.linkis.cli.common.entity.result.ExecutionResult; -import org.apache.linkis.cli.common.entity.result.ResultHandler; -import org.apache.linkis.cli.common.entity.var.VarAccess; -import org.apache.linkis.cli.common.exception.LinkisClientRuntimeException; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.common.exception.handler.ExceptionHandler; -import org.apache.linkis.cli.core.constants.CommonConstants; -import org.apache.linkis.cli.core.exception.CommandException; -import org.apache.linkis.cli.core.exception.PropsException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; -import org.apache.linkis.cli.core.exception.handler.CommandExceptionHandler; -import org.apache.linkis.cli.core.exception.handler.DefaultExceptionHandler; -import org.apache.linkis.cli.core.interactor.command.CmdTemplateFactory; -import org.apache.linkis.cli.core.interactor.command.fitter.SingleTplFitter; -import org.apache.linkis.cli.core.interactor.command.parser.Parser; -import org.apache.linkis.cli.core.interactor.command.parser.SingleCmdParser; -import org.apache.linkis.cli.core.interactor.command.parser.result.ParseResult; -import org.apache.linkis.cli.core.interactor.properties.PropertiesLoader; -import org.apache.linkis.cli.core.interactor.properties.PropsFilesScanner; -import org.apache.linkis.cli.core.interactor.properties.StdPropsLoader; -import org.apache.linkis.cli.core.interactor.properties.reader.PropertiesReader; -import org.apache.linkis.cli.core.interactor.properties.reader.PropsFileReader; -import org.apache.linkis.cli.core.interactor.properties.reader.SysEnvReader; -import org.apache.linkis.cli.core.interactor.properties.reader.SysPropsReader; -import org.apache.linkis.cli.core.interactor.result.DefaultResultHandler; -import org.apache.linkis.cli.core.interactor.result.ExecutionResultImpl; -import org.apache.linkis.cli.core.interactor.result.ExecutionStatusEnum; -import org.apache.linkis.cli.core.interactor.validate.ParsedTplValidator; -import org.apache.linkis.cli.core.interactor.var.StdVarAccess; -import org.apache.linkis.cli.core.interactor.var.SysVarAccess; -import org.apache.linkis.cli.core.operator.JobOperatorBuilder; -import org.apache.linkis.cli.core.operator.JobOperatorFactory; -import org.apache.linkis.cli.core.present.PresentModeImpl; -import org.apache.linkis.cli.core.present.display.DisplayOperFactory; -import org.apache.linkis.cli.core.present.display.PlainTextFileWriter; -import org.apache.linkis.cli.core.present.display.StdOutWriter; -import org.apache.linkis.cli.core.utils.LogUtils; -import org.apache.linkis.cli.core.utils.SchedulerUtils; - -import org.apache.commons.lang3.StringUtils; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class LinkisClientApplication { - private static Logger logger = LoggerFactory.getLogger(LinkisClientApplication.class); - - /** - * generate Templates load env variables TODO: load version info - * - * @return PreparedData - */ - private static PreparedData prepare() throws LinkisClientRuntimeException { - /* - generate template - */ - CmdTemplate template = new UniversalCmdTemplate(); - CmdTemplateFactory.register(template); - /* - load env variables - */ - Map propertiesMap = new HashMap<>(); - PropertiesLoader loader = - new StdPropsLoader() - .addPropertiesReader(new SysPropsReader()) - .addPropertiesReader(new SysEnvReader()); - for (ClientProperties properties : loader.loadProperties()) { - propertiesMap.put(properties.getPropsId(), properties); - } - - return new PreparedData(propertiesMap); - } - - /** - * parse user input load user config load default config check if all inputs are ok - * - * @param args user input arguments - * @return ProcessedData - */ - private static ProcessedData processInput(String[] args, PreparedData preparedData) - throws Exception { - - if (preparedData == null) { - return null; - } - - /* - user input - */ - CmdTemplate template = CmdTemplateFactory.getTemplateCopy(LinkisCmdType.UNIVERSAL); - Parser parser = - new SingleCmdParser() - .setMapper(null) - .setTemplate(template) - .setFitter(new SingleTplFitter()); - - ParseResult result = parser.parse(args); - - ParsedTplValidator parsedTplValidator = new ParsedTplValidator(); - parsedTplValidator.doValidation(result.getParsedTemplate()); - - Params params = result.getParams(); - logger.debug("==========params============\n" + Utils.GSON.toJson(params)); - - /* - VarAccess for sys_prop, sys_env - */ - Map propertiesMap = preparedData.getPropertiesMap(); - VarAccess sysVarAccess = - new SysVarAccess() - .setSysProp(propertiesMap.get(CommonConstants.SYSTEM_PROPERTIES_IDENTIFIER)) - .setSysEnv(propertiesMap.get(CommonConstants.SYSTEM_ENV_IDENTIFIER)); - logger.debug("==========sys_var============\n" + Utils.GSON.toJson(sysVarAccess)); - - LogUtils.getInformationLogger() - .info( - "LogFile path: " - + sysVarAccess.getVar(String.class, AppKeys.LOG_PATH_KEY) - + "/" - + sysVarAccess.getVar(String.class, AppKeys.LOG_FILE_KEY)); - /* - default config, -Dconf.root & -Dconf.file specifies config path - */ - // scan config files given root path - String configPath = sysVarAccess.getVar(String.class, AppKeys.CLIENT_CONFIG_ROOT_KEY); - String defaultConfFileName = - sysVarAccess.getVarOrDefault( - String.class, AppKeys.DEFAULT_CONFIG_FILE_NAME_KEY, AppConstants.DEFAULT_CONFIG_NAME); - if (StringUtils.isBlank(configPath)) { - throw new PropsException( - "PRP0007", - ErrorLevel.ERROR, - CommonErrMsg.PropsLoaderErr, - "configuration root path specified by env variable: " - + AppKeys.CLIENT_CONFIG_ROOT_KEY - + " is empty."); - } - - List readersList = - new PropsFilesScanner().getPropsReaders(configPath); // +1 user config - /* - user defined config - */ - String userConfPath = null; - if (params.containsParam(AppKeys.LINKIS_CLIENT_USER_CONFIG)) { - userConfPath = - (String) params.getParamItemMap().get(AppKeys.LINKIS_CLIENT_USER_CONFIG).getValue(); - } - if (StringUtils.isNotBlank(userConfPath)) { - PropertiesReader reader = - new PropsFileReader() - .setPropsId(AppKeys.LINKIS_CLIENT_USER_CONFIG) - .setPropsPath(userConfPath); - readersList.add(reader); - } else { - LogUtils.getInformationLogger() - .info("User does not provide usr-configuration file. Will use default config"); - } - /* - load properties - */ - PropertiesLoader loader = - new StdPropsLoader() - .addPropertiesReaders(readersList.toArray(new PropertiesReader[readersList.size()])); - ClientProperties[] loaderResult = loader.loadProperties(); - for (ClientProperties properties : loaderResult) { - if (StringUtils.equals(properties.getPropsId(), AppKeys.LINKIS_CLIENT_USER_CONFIG)) { - for (Map.Entry prop : properties.entrySet()) { - if (StringUtils.startsWith( - (String) prop.getKey(), AppKeys.LINKIS_CLIENT_NONCUSTOMIZABLE)) { - throw new PropsException( - "PRP0007", - ErrorLevel.ERROR, - CommonErrMsg.PropsLoaderErr, - "User cannot specify non-customizable configuration: " + prop.getKey()); - } - } - } - propertiesMap.put(properties.getPropsId(), properties); - } - - /* - VarAccess for cmd, config - */ - VarAccess stdVarAccess = - new StdVarAccess() - .setCmdParams(params) - .setUserConf(propertiesMap.get(AppKeys.LINKIS_CLIENT_USER_CONFIG)) - .setDefaultConf(propertiesMap.get(defaultConfFileName)) - .init(); - logger.info("==========std_var============\n" + Utils.GSON.toJson(stdVarAccess)); - - /* - Prepare operator for accessing linkis - */ - JobOperatorBuilder builder = - new LinkisOperatorBuilder().setStdVarAccess(stdVarAccess).setSysVarAccess(sysVarAccess); - - JobOperatorFactory.register(AppKeys.REUSABLE_UJES_CLIENT, builder); - /* - Prepare DisplayOperator - */ - DisplayOperFactory.register(PresentModeImpl.STDOUT, new StdOutWriter()); - DisplayOperFactory.register(PresentModeImpl.TEXT_FILE, new PlainTextFileWriter()); - - return new ProcessedData( - AppConstants.DUMMY_CID, params.getCmdType(), stdVarAccess, sysVarAccess); - } - - /** - * submit job display result - * - * @return FinishedData - */ - private static FinishedData exec(ProcessedData data) throws Exception { - if (data == null) { - return null; - } - - ExecutionSuite suite = - ExecutionSuiteFactory.getSuite( - data.getCmdType(), data.getStdVarAccess(), data.getSysVarAccess()); - - /* - Get everything - */ - Map jobs = suite.getJobs(); - ResultHandler[] resultHandlers = suite.getResultHandlers(); - Execution execution = suite.getExecution(); - - /* - execute - */ - final Map jobsToKill = jobs; - Thread hook = new Thread(() -> execution.terminate(jobsToKill)); - if (jobsToKill != null && jobsToKill.size() != 0) { - Runtime.getRuntime().addShutdownHook(hook); - } - ExecutionResult result = execution.execute(jobs); - - Runtime.getRuntime().removeShutdownHook(hook); - - return new FinishedData(result, resultHandlers); - } - - public static void main(String[] args) { - - ExceptionHandler handler = new DefaultExceptionHandler(); - ProcessedData processedData = null; - FinishedData finishedData = null; - ExecutionResult executionResult = new ExecutionResultImpl(null, ExecutionStatusEnum.UNDEFINED); - PreparedData preparedData = null; - - try { - preparedData = prepare(); - } catch (Exception e) { - handler.handle(e); - executionResult.setExecutionStatus(ExecutionStatusEnum.FAILED); - } - - try { - processedData = processInput(args, preparedData); - } catch (CommandException ce) { - new CommandExceptionHandler().handle(ce); - executionResult.setExecutionStatus(ExecutionStatusEnum.FAILED); - } catch (Exception e) { - handler.handle(e); - executionResult.setExecutionStatus(ExecutionStatusEnum.FAILED); - } - - try { - finishedData = exec(processedData); - } catch (Exception e) { - handler.handle(e); - executionResult.setExecutionStatus(ExecutionStatusEnum.FAILED); - } - - if (finishedData != null) { - executionResult = finishedData.getExecutionResult(); - if (executionResult == null) { - executionResult = new ExecutionResultImpl(null, ExecutionStatusEnum.UNDEFINED); - } - if (executionResult.getException() != null) { - handler.handle(executionResult.getException()); - new DefaultResultHandler().process(executionResult); - } else { - if (finishedData.getResultHandlers() != null) { - for (ResultHandler resultHandler : finishedData.getResultHandlers()) { - if (resultHandler != null) { - resultHandler.process(executionResult); - } - } - } - } - } else { - executionResult.setExecutionStatus(ExecutionStatusEnum.FAILED); - new DefaultResultHandler().process(executionResult); - } - - SchedulerUtils.shutDown(); - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/data/FinishedData.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/data/FinishedData.java deleted file mode 100644 index 4438cf301b..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/data/FinishedData.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.application.data; - -import org.apache.linkis.cli.common.entity.result.ExecutionResult; -import org.apache.linkis.cli.common.entity.result.ResultHandler; - -public class FinishedData { - ExecutionResult executionResult; - ResultHandler[] resultHandlers; - - public FinishedData(ExecutionResult executionResult, ResultHandler[] resultHandlers) { - this.executionResult = executionResult; - this.resultHandlers = resultHandlers; - } - - public ExecutionResult getExecutionResult() { - return executionResult; - } - - public void setExecutionResult(ExecutionResult executionResult) { - this.executionResult = executionResult; - } - - public ResultHandler[] getResultHandlers() { - return resultHandlers; - } - - public void setResultHandlers(ResultHandler[] resultHandlers) { - this.resultHandlers = resultHandlers; - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/data/ProcessedData.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/data/ProcessedData.java deleted file mode 100644 index a69b7c4b63..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/data/ProcessedData.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.application.data; - -import org.apache.linkis.cli.common.entity.command.CmdType; -import org.apache.linkis.cli.common.entity.var.VarAccess; - -public class ProcessedData { - String cid; - CmdType cmdType; - VarAccess stdVarAccess; - VarAccess sysVarAccess; - - public ProcessedData( - String cid, CmdType cmdType, VarAccess stdVarAccess, VarAccess sysVarAccess) { - this.cid = cid; - this.cmdType = cmdType; - this.stdVarAccess = stdVarAccess; - this.sysVarAccess = sysVarAccess; - } - - public String getCid() { - return cid; - } - - public CmdType getCmdType() { - return cmdType; - } - - public VarAccess getStdVarAccess() { - return stdVarAccess; - } - - public VarAccess getSysVarAccess() { - return sysVarAccess; - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/LinkisManageJob.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/LinkisManageJob.java deleted file mode 100644 index 7dbcaca88d..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/LinkisManageJob.java +++ /dev/null @@ -1,522 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.application.interactor.job; - -import org.apache.linkis.cli.application.constants.AppConstants; -import org.apache.linkis.cli.application.interactor.job.data.LinkisJobData; -import org.apache.linkis.cli.application.interactor.job.data.LinkisLogData; -import org.apache.linkis.cli.application.interactor.job.data.LinkisResultData; -import org.apache.linkis.cli.application.interactor.job.desc.LinkisJobManDesc; -import org.apache.linkis.cli.application.interactor.job.subtype.LinkisManSubType; -import org.apache.linkis.cli.application.operator.ujes.LinkisJobOperator; -import org.apache.linkis.cli.application.utils.Utils; -import org.apache.linkis.cli.common.entity.job.JobData; -import org.apache.linkis.cli.common.entity.operator.JobOperator; -import org.apache.linkis.cli.common.exception.LinkisClientRuntimeException; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.constants.CommonConstants; -import org.apache.linkis.cli.core.exception.LinkisClientExecutionException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; -import org.apache.linkis.cli.core.interactor.job.*; -import org.apache.linkis.cli.core.utils.CommonUtils; -import org.apache.linkis.cli.core.utils.LogUtils; -import org.apache.linkis.cli.core.utils.SchedulerUtils; - -import org.apache.commons.lang3.StringUtils; - -import java.text.MessageFormat; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class LinkisManageJob extends LinkisJob - implements ManagableBackendJob, TerminatableJob, LogAccessibleJob, ResultAccessibleJob { - private static final Logger logger = LoggerFactory.getLogger(LinkisManageJob.class); - - private LinkisJobManDesc jobDesc; - private LinkisJobData data; - private TerminateToken terminateToken = new TerminateToken(); - - @Override - public LinkisJobOperator getJobOperator() { - if (!(super.getJobOperator() instanceof LinkisJobOperator)) { - throw new LinkisClientExecutionException( - "EXE0003", - ErrorLevel.ERROR, - CommonErrMsg.ExecutionInitErr, - "JobOperator of LinkisManageJob should be instance of LinkisJobOperator"); - } - return (LinkisJobOperator) super.getJobOperator(); - } - - @Override - public void setOperator(JobOperator operator) { - if (!(operator instanceof LinkisJobOperator)) { - throw new LinkisClientExecutionException( - "EXE0003", - ErrorLevel.ERROR, - CommonErrMsg.ExecutionInitErr, - "JobOperator of LinkisManageJob should be instance of LinkisJobOperator"); - } - super.setOperator(operator); - } - - @Override - public LinkisJobManDesc getJobDesc() { - return jobDesc; - } - - public void setJobDesc(LinkisJobManDesc jobDesc) { - this.jobDesc = jobDesc; - } - - @Override - public LinkisJobData getJobData() { - return data; - } - - public void setJobData(LinkisJobData data) { - this.data = data; - } - - @Override - public boolean isSuccess() { - return data.isSuccess(); - } - - @Override - public void doManage() throws LinkisClientRuntimeException { - LinkisManSubType subType = (LinkisManSubType) getSubType(); - if (!(subType instanceof LinkisManSubType)) { - throw new LinkisClientExecutionException( - "EXE0030", - ErrorLevel.ERROR, - CommonErrMsg.ExecutionErr, - "JobSubType is not instance of JobManSubType"); - } - switch (subType) { - case STATUS: - try { - data.updateByOperResult( - getJobOperator().queryJobInfo(jobDesc.getUser(), jobDesc.getJobID())); - if (data.getJobStatus() != null) { - data.setSuccess(true); - } - } catch (Exception e) { - data.setSuccess(false); - data.setException(e); - } - break; - // case JOB_DESC: - // result = jobManagableBackendExecutor.queryJobDesc(job); - // break; - case LOG: - try { - ((LinkisLogData) data).setIncLogMode(false); - startRetrieveLog(); - waitLogFin(); - data.setSuccess(true); - } catch (Exception e) { - data.setSuccess(false); - data.setException(e); - } - break; - case RESULT: - try { - startRetrieveResult(); - data.setSuccess(true); - } catch (LinkisClientExecutionException e) { - if (e.getCode().equals("EXE0037")) { - ((LinkisResultData) data).sendResultFin(); // inform listener to stop - data.setSuccess(true); - } else { - data.setSuccess(false); - data.setException(e); - } - LogUtils.getInformationLogger().warn(e.getMessage()); - } catch (Exception e) { - data.setSuccess(false); - data.setException(e); - LogUtils.getInformationLogger().warn(e.getMessage()); - } - break; - // case LIST: - // resultData = jobManExecutor.queryJobList(job); - // break; - case KILL: - doKill(); - break; - default: - throw new LinkisClientExecutionException( - "EXE0002", - ErrorLevel.ERROR, - CommonErrMsg.ExecutionErr, - "JobSubType + \"" + subType + "\" is not supported"); - } - } - - @Override - public void startRetrieveLog() { - if (jobDesc.getUser() == null || jobDesc.getJobID() == null) { - throw new LinkisClientExecutionException( - "EXE0036", ErrorLevel.ERROR, CommonErrMsg.ExecutionErr, "user or jobID is null"); - } - data.updateByOperResult(getJobOperator().queryJobInfo(jobDesc.getUser(), jobDesc.getJobID())); - startRetrieveLogInternal(data); - } - - public void waitLogFin() { - if (!(data instanceof LinkisLogData)) { - return; - } - int retry = 0; - int MAX_RETRY = 300; // wait for 10 minutes after job finish - while (retry++ < MAX_RETRY) { - if (((LinkisLogData) data).logFinReceived()) { - return; - } - CommonUtils.doSleepQuietly(CommonConstants.JOB_QUERY_SLEEP_MILLS); - } - } - - public void startRetrieveLogInternal(JobData jobData) { - if (!(jobData instanceof LinkisLogData)) { - throw new LinkisClientExecutionException( - "EXE0034", ErrorLevel.ERROR, CommonErrMsg.ExecutionErr, "JobData is not LinkisLogData"); - } - if (jobData.getUser() == null || jobData.getJobID() == null) { - throw new LinkisClientExecutionException( - "EXE0036", ErrorLevel.ERROR, CommonErrMsg.ExecutionErr, "user or jobID is null"); - } - LinkisLogData logData = (LinkisLogData) jobData; - if (logData.getJobStatus() != null) { - try { - Thread logConsumer = new Thread(() -> logData.notifyLogListener(), "Log-Consumer"); - Thread logRetriever = new Thread(() -> queryLogLoop(logData), "Log-Retriever"); - SchedulerUtils.getCachedThreadPoolExecutor().execute(logRetriever); - SchedulerUtils.getCachedThreadPoolExecutor().execute(logConsumer); - } catch (Exception e) { - logger.warn("Failed to retrieve log", e); - } - } - } - - public void queryLogLoop(LinkisLogData data) { - int curLogIdx; - int nextLogIdx; - boolean hasNext = true; - int retryCnt = 0; - final int MAX_RETRY = 12; // continues fails for 90s, then exit thread - try { - while (hasNext) { - curLogIdx = data.getNextLogLineIdx() == null ? 0 : data.getNextLogLineIdx(); - try { - data.updateByOperResult(getJobOperator().queryJobInfo(data.getUser(), data.getJobID())); - queryJobLogFromLine(data, curLogIdx); - } catch (Exception e) { - logger.error("Cannot get inc-log:", e); - // and yes sometimes server may not be able to prepare persisted-log - retryCnt++; - if (retryCnt >= MAX_RETRY) { - logger.error( - "Continuously failing to query inc-log for " - + MAX_RETRY * (MAX_RETRY + 2) * 500 / 1000 - + "s. Will no longer try to query log", - e); - break; - } - Utils.doSleepQuietly(500l + 500l * retryCnt); // maybe server problem. sleep longer - continue; - } - retryCnt = 0; - nextLogIdx = data.getNextLogLineIdx() == null ? curLogIdx : data.getNextLogLineIdx(); - if (data.isIncLogMode()) { - hasNext = data.hasNextLogLine() == null ? curLogIdx < nextLogIdx : data.hasNextLogLine(); - } else { - hasNext = curLogIdx < nextLogIdx; - } - if (curLogIdx >= nextLogIdx) { - String msg = - MessageFormat.format( - "Job is still running, status={0}, progress={1}", - data.getJobStatus(), String.valueOf(data.getJobProgress() * 100) + "%"); - logger.info(msg); - } - Utils.doSleepQuietly(AppConstants.JOB_QUERY_SLEEP_MILLS); - } - } catch (Exception e) { - logger.error("Something goes wrong. Job Log may be incomplete", e); - } finally { - data.sendLogFin(); - } - } - - private void queryJobLogFromLine(LinkisLogData data, int fromLine) - throws LinkisClientRuntimeException { - if (!data.getJobStatus().isJobFinishedState()) { - try { - data.updateByOperResult( - getJobOperator() - .queryRunTimeLogFromLine( - data.getUser(), data.getJobID(), data.getExecID(), fromLine)); - } catch (Exception e) { - // job is finished while we start query log(but request is not send). - // then probably server cache is gone and we got a exception here. - // however we cannot know if this happens based on the exception message - logger.warn( - "Caught exception when querying runtime-log. Probably server-side has close stream. Will try openLog api if Job is completed.", - e); - if (data.getJobStatus().isJobFinishedState()) { - CommonUtils.doSleepQuietly(500l); - data.updateByOperResult( - getJobOperator() - .queryPersistedLogFromLine( - data.getUser(), data.getJobID(), data.getExecID(), fromLine)); - } - } - } else { - try { - data.updateByOperResult( - getJobOperator() - .queryPersistedLogFromLine( - data.getLogPath(), data.getUser(), data.getJobID(), fromLine)); - } catch (Exception e) { - logger.error("Cannot get persisted-inc-log:", e); - // and yes sometimes server may not be able to prepare persisted-log - throw e; - } - } - } - - @Override - public void startRetrieveResult() { - if (!(data instanceof LinkisResultData)) { - throw new LinkisClientExecutionException( - "EXE0034", - ErrorLevel.ERROR, - CommonErrMsg.ExecutionErr, - "JobData is not LinkisResultData"); - } - if (jobDesc.getUser() == null || jobDesc.getJobID() == null) { - throw new LinkisClientExecutionException( - "EXE0036", ErrorLevel.ERROR, CommonErrMsg.ExecutionErr, "user or jobID is null"); - } - data.updateByOperResult(getJobOperator().queryJobInfo(jobDesc.getUser(), jobDesc.getJobID())); - if (data.getJobStatus() == null) { - throw new LinkisClientExecutionException( - "EXE0038", ErrorLevel.ERROR, CommonErrMsg.ExecutionErr, "jobStatus is null"); - } - LinkisResultData resultData = (LinkisResultData) data; - if (!resultData.getJobStatus().isJobSuccess() - || StringUtils.isBlank(resultData.getResultLocation())) { - resultData.updateByOperResult( - getJobOperator().queryJobInfo(resultData.getUser(), resultData.getJobID())); - } - if (!resultData.getJobStatus().isJobSuccess()) { - // throw new LinkisClientExecutionException("EXE0035", ErrorLevel.ERROR, - // CommonErrMsg.ExecutionErr, "Job status is not success but \'" + - // resultData.getJobStatus() + "\'. Will not try to retrieve any Result"); - LogUtils.getInformationLogger() - .info( - "Job status is not success but \'" - + resultData.getJobStatus() - + "\'. Will not try to retrieve any Result"); - resultData.sendResultFin(); // inform listener to stop - return; - } - if (StringUtils.isBlank(resultData.getResultLocation())) { - throw new LinkisClientExecutionException( - "EXE0037", - ErrorLevel.WARN, - CommonErrMsg.ExecutionErr, - "Got blank ResultLocation from server. Job may not have result-set. Will not try to retrieve any Result"); - } - - resultData.updateByOperResult( - getJobOperator() - .queryResultSetPaths( - resultData.getUser(), resultData.getJobID(), resultData.getResultLocation())); - if (resultData.getResultSetPaths() == null || resultData.getResultSetPaths().length == 0) { - String msg = "Your job got no result."; - logger.warn(msg); - resultData.sendResultFin(); // inform listener to stop - resultData.setHasResult(false); - return; - } - - try { - resultData.setHasResult(true); - Thread resultRetriever = new Thread(() -> queryResultLoop(resultData), "Result-Retriever"); - SchedulerUtils.getCachedThreadPoolExecutor().execute(resultRetriever); - } catch (Exception e) { - logger.error("Failed to retrieve result", e); - throw e; - } - } - - public void queryResultLoop(LinkisResultData data) { - boolean hasNext = true; - int retryCnt = 0; - final int MAX_RETRY = 30; // continues fails for 250s, then exit - int idx = 0; - try { - while (hasNext) { - try { - hasNext = queryOneResult(data, idx); - } catch (LinkisClientRuntimeException e) { - logger.error("Cannot get result:", e); - retryCnt++; - if (retryCnt >= MAX_RETRY) { - logger.error( - "Continuously failing to query result for " - + MAX_RETRY * (MAX_RETRY + 2) * 500 / 1000 - + "s. Will no longer try to query result", - e); - return; - } else { - hasNext = true; - } - Utils.doSleepQuietly(500l + 500l * retryCnt); // maybe server problem. sleep longer - continue; - } - idx++; - } - } catch (Exception e) { - logger.error("Something goes wrong. Job Result may be incomplete", e); - throw e; - } finally { - data.sendResultFin(); - } - } - - private boolean queryOneResult(LinkisResultData data, int idxResultSet) { - Integer curPage = 1; - boolean hasNextResult = true; - boolean hasNextPage = true; - while (hasNextPage) { - data.updateByOperResult( - getJobOperator() - .queryResultSetGivenResultSetPath( - data.getResultSetPaths(), - idxResultSet, - data.getUser(), - curPage, - AppConstants.RESULTSET_PAGE_SIZE)); - if (data.hasNextResultPage() == null) { - throw new LinkisClientExecutionException( - "EXE0040", - ErrorLevel.ERROR, - CommonErrMsg.ExecutionResultErr, - "Something foes wrong. Got null as \'hasNextPage\'."); - } - hasNextPage = data.hasNextResultPage(); - - curPage++; - hasNextResult = idxResultSet + 1 < data.getResultSetPaths().length; - } - return hasNextResult; - } - - public void doKill() { - data.updateByOperResult(getJobOperator().queryJobInfo(jobDesc.getUser(), jobDesc.getJobID())); - if (data.getUser() == null || data.getJobID() == null) { - throw new LinkisClientExecutionException( - "EXE0036", ErrorLevel.ERROR, CommonErrMsg.ExecutionErr, "user or jobID is null"); - } - if (data.getJobStatus() == null) { - throw new LinkisClientExecutionException( - "EXE0038", ErrorLevel.ERROR, CommonErrMsg.ExecutionErr, "jobStatus is null"); - } - String msg; - if (data.getJobStatus().isJobCancelled()) { - msg = "Kill job aborted: Job has already been canceled."; - data.setSuccess(false); - data.setMessage(msg); - } else if (data.getJobStatus().isJobFinishedState()) { - msg = "Kill job aborted: Job is already in finished-state(SUCCEED/FAILED)."; - data.setSuccess(false); - data.setMessage(msg); - // throw new LinkisClientExecutionException(JobStatus.FAILED, "EXE0004", - // ErrorLevel.ERROR, CommonErrMsg.ExecutionErr, msg); - } else { - try { - data.updateByOperResult( - getJobOperator().queryJobInfo(jobDesc.getUser(), jobDesc.getJobID())); - data.updateByOperResult( - getJobOperator().kill(data.getUser(), data.getJobID(), data.getExecID())); - } catch (Exception e) { - data.setSuccess(false); - data.setMessage("Exception thrown when trying to send kill request"); - data.setException(e); - } - msg = "Kill request has been sent"; - LogUtils.getPlaintTextLogger().info(msg); - int retryCnt = 0; - final int MAX_RETRY = 5 * 6; - while (!data.getJobStatus().isJobFinishedState() && !data.getJobStatus().isJobCancelled()) { - CommonUtils.doSleepQuietly(CommonConstants.JOB_QUERY_SLEEP_MILLS); - try { - data.updateByOperResult( - getJobOperator().queryJobInfo(jobDesc.getUser(), jobDesc.getJobID())); - retryCnt = 0; // if exception then will not go here - } catch (Exception e) { - retryCnt++; - CommonUtils.doSleepQuietly(5 * CommonConstants.JOB_QUERY_SLEEP_MILLS); - if (retryCnt >= MAX_RETRY) { - data.setSuccess(false); - data.setMessage( - MessageFormat.format( - "After send kill. Client cannot get jobStatus from server continuously for {0} seconds. Client aborted. Assume kill failed! Error message: \n", - MAX_RETRY * 5 * CommonConstants.JOB_QUERY_SLEEP_MILLS)); - data.setException(e); - return; - } - } - } - if (data.getJobStatus().isJobFinishedState() && !data.getJobStatus().isJobCancelled()) { - msg = "Kill Failed: Job Current status: " + data.getJobStatus(); - data.setSuccess(false); - data.setMessage(msg); - // throw new LinkisClientExecutionException(JobStatus.FAILED, - // "EXE0004", ErrorLevel.ERROR, CommonErrMsg.ExecutionErr, msg); - } else if (data.getJobStatus().isJobCancelled()) { - msg = - MessageFormat.format( - "Kill successful: jobId={0}, status={1}.", data.getJobID(), data.getJobStatus()); - data.setSuccess(true); - data.setMessage(msg); - // LogUtils.getPlaintTextLogger().info(msg); - } - } - return; - } - - @Override - public TerminateToken getTerminateToken() { - return terminateToken; - } - - public void setTerminateToken(TerminateToken terminateToken) { - this.terminateToken = terminateToken; - } - - @Override - public void terminate() throws LinkisClientRuntimeException { - return; - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/LinkisOnceJob.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/LinkisOnceJob.java deleted file mode 100644 index 505b08cf26..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/LinkisOnceJob.java +++ /dev/null @@ -1,308 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.application.interactor.job; - -import org.apache.linkis.cli.application.constants.AppConstants; -import org.apache.linkis.cli.application.interactor.job.data.LinkisJobData; -import org.apache.linkis.cli.application.interactor.job.data.LinkisLogData; -import org.apache.linkis.cli.application.interactor.job.data.LinkisOnceJobData; -import org.apache.linkis.cli.application.interactor.job.desc.LinkisOnceDesc; -import org.apache.linkis.cli.application.utils.Utils; -import org.apache.linkis.cli.common.entity.job.JobDescription; -import org.apache.linkis.cli.common.entity.operator.JobOperator; -import org.apache.linkis.cli.common.exception.LinkisClientRuntimeException; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.constants.CommonConstants; -import org.apache.linkis.cli.core.exception.LinkisClientExecutionException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; -import org.apache.linkis.cli.core.interactor.job.*; -import org.apache.linkis.cli.core.utils.CommonUtils; -import org.apache.linkis.cli.core.utils.LogUtils; -import org.apache.linkis.cli.core.utils.SchedulerUtils; - -import java.text.MessageFormat; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class LinkisOnceJob extends LinkisJob - implements ManagableBackendJob, - LogAccessibleJob, - ResultAccessibleJob, - AsyncBackendJob, - TerminatableJob { - - private static final Logger logger = LoggerFactory.getLogger(LinkisOnceJob.class); - - private LinkisOnceDesc jobDesc; - private LinkisOnceJobData data; - private TerminateToken terminateToken = new TerminateToken(); - private Boolean isAsync = false; - - public void setAsync(Boolean async) { - isAsync = async; - } - - public Boolean isAsync() { - return isAsync; - } - - @Override - public JobDescription getJobDesc() { - return jobDesc; - } - - public void setJobDesc(LinkisOnceDesc desc) { - this.jobDesc = desc; - } - - @Override - public TerminateToken getTerminateToken() { - return terminateToken; - } - - @Override - public LinkisJobData getJobData() { - return data; - } - - public void setJobData(LinkisOnceJobData data) { - this.data = data; - } - - @Override - public JobOperator getJobOperator() { - return null; - } - - /** AsyncBackendJob */ - @Override - public void submit() throws LinkisClientRuntimeException { - StringBuilder infoBuilder = new StringBuilder(); - infoBuilder - .append("connecting to linkis gateway:") - .append(data.getOnceJobAdapter().getServerUrl()); - LogUtils.getInformationLogger().info(infoBuilder.toString()); - data.getOnceJobAdapter().submit(); - data.getOnceJobAdapter().updateStatus(); - infoBuilder.setLength(0); - infoBuilder - .append("JobId:") - .append(data.getJobID()) - .append(System.lineSeparator()) - .append("ExecId:") - .append(data.getExecID()); - LogUtils.getPlaintTextLogger().info(infoBuilder.toString()); - if (isAsync) { - data.setSuccess(data.getJobStatus() != null && data.getJobStatus().isJobSubmitted()); - } - } - - @Override - public void updateJobStatus() throws LinkisClientRuntimeException { - if (!data.getJobStatus().isJobFinishedState()) { - data.getOnceJobAdapter().updateStatus(); - String log2 = - "\n---------------------------------------------------\n" - + "\ttask " - + data.getJobID() - + " status is " - + data.getJobStatus() - + ", progress : " - + data.getJobProgress() - + "\n---------------------------------------------------"; - logger.info(log2); - } - } - - @Override - public void waitJobComplete() throws LinkisClientRuntimeException { - data.getOnceJobAdapter().waitForComplete(); - updateJobStatus(); - data.setSuccess(data.getJobStatus() != null && data.getJobStatus().isJobSuccess()); - waitIncLogComplete(data); - } - - /** TerminatableJob */ - @Override - public void terminate() throws LinkisClientRuntimeException { - terminateToken.setTerminate(); - doKill(); - } - - @Override - public void startRetrieveResult() { - // TODO:wait for OnceJob to support this feature - data.sendResultFin(); - } - - @Override - public void startRetrieveLog() { - data.setIncLogMode(true); - startRetrieveLogInternal(data); - } - - public void startRetrieveLogInternal(LinkisOnceJobData jobData) { - if (!(jobData instanceof LinkisLogData)) { - throw new LinkisClientExecutionException( - "EXE0034", ErrorLevel.ERROR, CommonErrMsg.ExecutionErr, "JobData is not LinkisLogData"); - } - if (jobData.getUser() == null || jobData.getJobID() == null) { - throw new LinkisClientExecutionException( - "EXE0036", ErrorLevel.ERROR, CommonErrMsg.ExecutionErr, "user or jobID is null"); - } - LinkisOnceJobData logData = jobData; - if (logData.getJobStatus() != null) { - try { - Thread logConsumer = new Thread(() -> logData.notifyLogListener(), "Log-Consumer"); - Thread logRetriever = new Thread(() -> queryLogLoop(logData), "Log-Retriever"); - SchedulerUtils.getCachedThreadPoolExecutor().execute(logRetriever); - SchedulerUtils.getCachedThreadPoolExecutor().execute(logConsumer); - } catch (Exception e) { - logger.warn("Failed to retrieve log", e); - } - } - } - - private void queryJobLogOneIteration(LinkisOnceJobData data) throws LinkisClientRuntimeException { - try { - data.getOnceJobAdapter().queryJobLogOneIteration(); - // - // data.updateByOperResult(getJobOperator().queryRunTimeLogFromLine(data.getUser(), - // data.getJobID(), data.getExecID(), fromLine)); - } catch (Exception e) { - // job is finished while we start query log(but request is not send). - // then probably server cache is gone and we got a exception here. - // however we cannot know if this happens based on the exception message - logger.warn( - "Caught exception when querying runtime-log. Probably server-side has close stream. Will try openLog api if Job is completed.", - e); - if (data.getJobStatus().isJobFinishedState()) { - CommonUtils.doSleepQuietly(500l); - // - // data.updateByOperResult(getJobOperator().queryPersistedLogFromLine(data.getUser(), - // data.getJobID(), data.getExecID(), fromLine)); - } - } - } - - public void queryLogLoop(LinkisOnceJobData data) { - boolean hasNext = true; - int retryCnt = 0; - final int MAX_RETRY = 12; // continues fails for 90s, then exit thread - try { - while (hasNext) { - try { - queryJobLogOneIteration(data); - } catch (Exception e) { - logger.error("Cannot get inc-log:", e); - // and yes sometimes server may not be able to prepare persisted-log - retryCnt++; - if (retryCnt >= MAX_RETRY) { - logger.error( - "Continuously failing to query inc-log for " - + MAX_RETRY * (MAX_RETRY + 2) * 500 / 1000 - + "s. Will no longer try to query log", - e); - break; - } - Utils.doSleepQuietly(500l + 500l * retryCnt); // maybe server problem. sleep longer - continue; - } - retryCnt = 0; - if (data.isIncLogMode()) { - hasNext = - data.hasNextLogLine() == null - ? !data.getJobStatus().isJobFinishedState() - : data.hasNextLogLine(); - } else { - hasNext = false; - } - if (hasNext) { - String msg = - MessageFormat.format( - "Job is still running, status={0}, progress={1}", - data.getJobStatus(), String.valueOf(data.getJobProgress() * 100) + "%"); - logger.info(msg); - } - Utils.doSleepQuietly(AppConstants.JOB_QUERY_SLEEP_MILLS); - } - } catch (Exception e) { - logger.error("Something goes wrong. Job Log may be incomplete", e); - } finally { - data.sendLogFin(); - } - } - - private void waitIncLogComplete(LinkisJobData data) { - if (!(data instanceof LinkisOnceJobData)) { - return; - } - int retry = 0; - int MAX_RETRY = 300; // wait for 10 minutes after job finish - while (retry++ < MAX_RETRY) { - if (((LinkisOnceJobData) data).logFinReceived()) { - return; - } - CommonUtils.doSleepQuietly(CommonConstants.JOB_QUERY_SLEEP_MILLS); - } - String msg = - "Job is in Finished state(SUCCEED/FAILED/CANCELED) but client keep querying inclog for " - + (MAX_RETRY * CommonConstants.JOB_QUERY_SLEEP_MILLS / 1000) - + "seconds. Execution ends forcefully. Next will try handle execution result."; - logger.warn(msg); - LogUtils.getInformationLogger().warn(msg); - } - - // /** - // * LogAccessibleJob - // */ - // @Override - // public void startRetrieveLog() { - - // } - - /** ManagableBackendJob */ - @Override - public void doManage() throws LinkisClientRuntimeException {} - - @Override - public boolean isSuccess() { - return data.isSuccess(); - } - - private void doKill() { - String msg; - if (data.getJobStatus().isJobCancelled()) { - msg = "Kill job aborted: Job is failed or has already been canceled."; - data.setSuccess(false); - data.setMessage(msg); - } else if (data.getJobStatus().isJobFinishedState()) { - msg = "Kill job aborted: Job is already in finished-state(SUCCEED/FAILED)."; - data.setSuccess(false); - data.setMessage(msg); - // throw new LinkisClientExecutionException(JobStatus.FAILED, "EXE0004", - // ErrorLevel.ERROR, CommonErrMsg.ExecutionErr, msg); - } else { - data.getOnceJobAdapter().kill(); - updateJobStatus(); - data.setSuccess(true); - data.setMessage("successfully killed job"); - } - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/LinkisSubmitJob.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/LinkisSubmitJob.java deleted file mode 100644 index d1e71e4e8d..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/LinkisSubmitJob.java +++ /dev/null @@ -1,282 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.application.interactor.job; - -import org.apache.linkis.cli.application.interactor.job.data.LinkisJobData; -import org.apache.linkis.cli.application.interactor.job.data.LinkisLogData; -import org.apache.linkis.cli.application.interactor.job.data.LinkisResultData; -import org.apache.linkis.cli.application.interactor.job.desc.LinkisJobManDesc; -import org.apache.linkis.cli.application.interactor.job.desc.LinkisSubmitDesc; -import org.apache.linkis.cli.application.operator.ujes.LinkisJobOperator; -import org.apache.linkis.cli.common.entity.operator.JobOperator; -import org.apache.linkis.cli.common.exception.LinkisClientRuntimeException; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.constants.CommonConstants; -import org.apache.linkis.cli.core.exception.LinkisClientExecutionException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; -import org.apache.linkis.cli.core.interactor.job.*; -import org.apache.linkis.cli.core.utils.CommonUtils; -import org.apache.linkis.cli.core.utils.LogUtils; - -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.exception.ExceptionUtils; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class LinkisSubmitJob extends LinkisJob - implements AsyncBackendJob, LogAccessibleJob, ResultAccessibleJob, TerminatableJob { - private static final Logger logger = LoggerFactory.getLogger(LinkisSubmitJob.class); - - private LinkisSubmitDesc jobDesc; - private LinkisJobData data; - private TerminateToken terminateToken = new TerminateToken(); - private LinkisManageJob manageJob = new LinkisManageJob(); - private Boolean isAsync = false; - - public void setAsync(Boolean async) { - isAsync = async; - } - - public Boolean isAsync() { - return isAsync; - } - - @Override - public LinkisJobOperator getJobOperator() { - if (!(super.getJobOperator() instanceof LinkisJobOperator)) { - throw new LinkisClientExecutionException( - "EXE0003", - ErrorLevel.ERROR, - CommonErrMsg.ExecutionInitErr, - "JobOperator of LinkisManageJob should be instance of LinkisJobOperator"); - } - return (LinkisJobOperator) super.getJobOperator(); - } - - @Override - public void setOperator(JobOperator operator) { - if (!(operator instanceof LinkisJobOperator)) { - throw new LinkisClientExecutionException( - "EXE0003", - ErrorLevel.ERROR, - CommonErrMsg.ExecutionInitErr, - "JobOperator of LinkisManageJob should be instance of LinkisJobOperator"); - } - manageJob.setOperator(operator); - super.setOperator(operator); - } - - @Override - public LinkisSubmitDesc getJobDesc() { - return jobDesc; - } - - public void setJobDesc(LinkisSubmitDesc jobDesc) { - this.jobDesc = jobDesc; - } - - @Override - public LinkisJobData getJobData() { - return data; - } - - public void setJobData(LinkisJobData data) { - manageJob.setJobData(data); - this.data = data; - } - - @Override - public TerminateToken getTerminateToken() { - return terminateToken; - } - - public void setTerminateToken(TerminateToken terminateToken) { - this.terminateToken = terminateToken; - } - - @Override - public void submit() throws LinkisClientRuntimeException { - StringBuilder infoBuilder = new StringBuilder(); - infoBuilder.append("connecting to linkis gateway:").append(getJobOperator().getServerUrl()); - LogUtils.getInformationLogger().info(infoBuilder.toString()); - data.updateByOperResult(getJobOperator().submit(jobDesc)); - CommonUtils.doSleepQuietly(2000l); - LinkisJobManDesc jobManDesc = new LinkisJobManDesc(); - jobManDesc.setJobId(data.getJobID()); - jobManDesc.setUser(data.getUser()); - manageJob.setJobDesc(jobManDesc); - data.updateByOperResult(getJobOperator().queryJobInfo(data.getUser(), data.getJobID())); - infoBuilder.setLength(0); - infoBuilder - .append("JobId:") - .append(data.getJobID()) - .append(System.lineSeparator()) - .append("TaskId:") - .append(data.getJobID()) - .append(System.lineSeparator()) - .append("ExecId:") - .append(data.getExecID()); - LogUtils.getPlaintTextLogger().info(infoBuilder.toString()); - if (isAsync) { - data.setSuccess(data.getJobStatus() != null && data.getJobStatus().isJobSubmitted()); - } - } - - @Override - public void updateJobStatus() throws LinkisClientRuntimeException { - data.updateByOperResult(getJobOperator().queryJobInfo(data.getUser(), data.getJobID())); - getJobOperator().queryJobStatus(data.getUser(), data.getJobID(), data.getExecID()); - String log2 = - "\n---------------------------------------------------\n" - + "\ttask " - + data.getJobID() - + " status is " - + data.getJobStatus() - + ", progress : " - + data.getJobProgress() - + "\n---------------------------------------------------"; - logger.info(log2); - } - - @Override - public void waitJobComplete() throws LinkisClientRuntimeException { - int retryCnt = 0; - final int MAX_RETRY = 30; - while (!data.getJobStatus().isJobFinishedState()) { - // query progress - try { - data.updateByOperResult(getJobOperator().queryJobInfo(data.getUser(), data.getJobID())); - getJobOperator().queryJobStatus(data.getUser(), data.getJobID(), data.getExecID()); - } catch (Exception e) { - logger.warn("", e); - retryCnt++; - if (retryCnt >= MAX_RETRY) { - throw new LinkisClientExecutionException( - "EXE0013", - ErrorLevel.ERROR, - CommonErrMsg.ExecutionErr, - "Cannot get jobStatus from server continuously for {0} seconds. Client aborted! Error message: \n", - MAX_RETRY * 5 * CommonConstants.JOB_QUERY_SLEEP_MILLS, - e); - } - CommonUtils.doSleepQuietly( - 5 * CommonConstants.JOB_QUERY_SLEEP_MILLS); // maybe server problem. sleep - // longer - continue; - } - retryCnt = 0; // reset counter - checkJobAvailability(data); - CommonUtils.doSleepQuietly(CommonConstants.JOB_QUERY_SLEEP_MILLS); - } - data.setSuccess(data.getJobStatus() != null && data.getJobStatus().isJobSuccess()); - waitIncLogComplete(data); - } - - private void waitIncLogComplete(LinkisJobData data) { - if (!(data instanceof LinkisLogData)) { - return; - } - int retry = 0; - int MAX_RETRY = 300; // wait for 10 minutes after job finish - while (retry++ < MAX_RETRY) { - if (((LinkisLogData) data).logFinReceived()) { - return; - } - CommonUtils.doSleepQuietly(CommonConstants.JOB_QUERY_SLEEP_MILLS); - } - String msg = - "Job is in Finished state(SUCCEED/FAILED/CANCELED) but client keep querying inclog for " - + (MAX_RETRY * CommonConstants.JOB_QUERY_SLEEP_MILLS / 1000) - + "seconds. Execution ends forcefully. Next will try handle execution result."; - logger.warn(msg); - LogUtils.getInformationLogger().warn(msg); - } - - @Override - public void startRetrieveResult() { - try { - manageJob.startRetrieveResult(); - data.setSuccess(true); - } catch (LinkisClientExecutionException e) { - if (e.getCode().equals("EXE0037")) { - data.setSuccess(true); - LogUtils.getInformationLogger().warn(e.getMessage()); - } else { - data.setSuccess(false); - data.setException(e); - } - ((LinkisResultData) data).sendResultFin(); // inform listener to stop - } catch (Exception e) { - data.setSuccess(false); - data.setException(e); - ((LinkisResultData) data).sendResultFin(); // inform listener to stop - } - } - - @Override - public void startRetrieveLog() { - if (!(data instanceof LinkisLogData)) { - throw new LinkisClientExecutionException( - "EXE0034", ErrorLevel.ERROR, CommonErrMsg.ExecutionErr, "JobData is not LinkisLogData"); - } - LinkisLogData dataCopy; - try { - dataCopy = ((LinkisLogData) data).clone(); // make a copy to avoid race condition - } catch (CloneNotSupportedException e) { - throw new LinkisClientExecutionException( - "EXE0035", ErrorLevel.ERROR, CommonErrMsg.ExecutionErr, "logData is not Cloneable", e); - } - dataCopy.setIncLogMode(true); - manageJob.startRetrieveLogInternal(dataCopy); - } - - @Override - public void terminate() throws LinkisClientRuntimeException { - terminateToken.setTerminate(); - // kill job if job is submitted - if (StringUtils.isNotBlank(data.getJobID())) { - System.out.println("\nKilling job: " + data.getJobID()); - try { - manageJob.doKill(); - if (data.getJobStatus().isJobCancelled()) { - System.out.println("Successfully killed job: " + data.getJobID() + " on exit"); - } else { - System.out.println( - "Failed to kill job: " - + data.getJobID() - + " on exit. Current job status: " - + data.getJobStatus()); - } - } catch (Exception e) { - System.out.println("Failed to kill job: " + data.getJobID() + " on exit"); - System.out.println(ExceptionUtils.getStackTrace(e)); - } - } - } - - private void checkJobAvailability(LinkisJobData data) throws LinkisClientRuntimeException { - if (data.getJobStatus().isJobAbnormalStatus()) { - throw new LinkisClientExecutionException( - "EXE0006", - ErrorLevel.ERROR, - CommonErrMsg.ExecutionErr, - "Job is in abnormal status: " + CommonUtils.GSON.toJson(data)); - } - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/builder/LinkisManageJobBuilder.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/builder/LinkisManageJobBuilder.java deleted file mode 100644 index ef5648b993..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/builder/LinkisManageJobBuilder.java +++ /dev/null @@ -1,140 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.application.interactor.job.builder; - -import org.apache.linkis.cli.application.constants.AppKeys; -import org.apache.linkis.cli.application.interactor.job.LinkisManageJob; -import org.apache.linkis.cli.application.interactor.job.data.LinkisJobData; -import org.apache.linkis.cli.application.interactor.job.data.LinkisJobDataImpl; -import org.apache.linkis.cli.application.interactor.job.desc.LinkisJobManDesc; -import org.apache.linkis.cli.application.observer.listener.LinkisClientListener; -import org.apache.linkis.cli.application.operator.ujes.LinkisJobOperator; -import org.apache.linkis.cli.application.utils.ExecutionUtils; -import org.apache.linkis.cli.common.entity.present.PresentWay; -import org.apache.linkis.cli.common.exception.LinkisClientRuntimeException; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; -import org.apache.linkis.cli.core.interactor.job.JobBuilder; -import org.apache.linkis.cli.core.operator.JobOperatorFactory; -import org.apache.linkis.cli.core.present.PresentModeImpl; -import org.apache.linkis.cli.core.present.PresentWayImpl; - -import org.apache.commons.lang3.StringUtils; - -import java.util.HashSet; -import java.util.Set; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class LinkisManageJobBuilder extends JobBuilder { - private static Logger logger = LoggerFactory.getLogger(LinkisSubmitJobBuilder.class); - - LinkisClientListener logListener; - - public LinkisManageJobBuilder setLogListener(LinkisClientListener observer) { - this.logListener = observer; - return this; - } - - @Override - protected LinkisJobManDesc buildJobDesc() { - LinkisJobManDesc desc = new LinkisJobManDesc(); - String osUser = sysVarAccess.getVar(String.class, AppKeys.LINUX_USER_KEY); - String[] adminUsers = StringUtils.split(AppKeys.ADMIN_USERS, ','); - Set adminSet = new HashSet<>(); - for (String admin : adminUsers) { - adminSet.add(admin); - } - String submitUsr = ExecutionUtils.getSubmitUser(stdVarAccess, osUser, adminSet); - - String jobId = null; - if (stdVarAccess.hasVar(AppKeys.LINKIS_CLIENT_KILL_OPT)) { - jobId = stdVarAccess.getVar(String.class, AppKeys.LINKIS_CLIENT_KILL_OPT); - } else if (stdVarAccess.hasVar(AppKeys.LINKIS_CLIENT_STATUS_OPT)) { - jobId = stdVarAccess.getVar(String.class, AppKeys.LINKIS_CLIENT_STATUS_OPT); - } else if (stdVarAccess.hasVar(AppKeys.LINKIS_CLIENT_DESC_OPT)) { - jobId = stdVarAccess.getVar(String.class, AppKeys.LINKIS_CLIENT_DESC_OPT); - } else if (stdVarAccess.hasVar(AppKeys.LINKIS_CLIENT_LOG_OPT)) { - jobId = stdVarAccess.getVar(String.class, AppKeys.LINKIS_CLIENT_LOG_OPT); - } else if (stdVarAccess.hasVar(AppKeys.LINKIS_CLIENT_RESULT_OPT)) { - jobId = stdVarAccess.getVar(String.class, AppKeys.LINKIS_CLIENT_RESULT_OPT); - } else if (stdVarAccess.hasVar(AppKeys.LINKIS_CLIENT_LIST_OPT)) { - jobId = stdVarAccess.getVar(String.class, AppKeys.LINKIS_CLIENT_LIST_OPT); - } - - desc.setJobId(jobId); - desc.setUser(submitUsr); - return desc; - } - - @Override - protected LinkisJobData buildJobData() { - LinkisJobDataImpl data = new LinkisJobDataImpl(); - if (logListener == null) { - logger.warn("logListener is not registered, will not be able to display log"); - } else { - data.registerincLogListener(logListener); - } - return data; - } - - @Override - protected LinkisJobOperator buildJobOperator() { - LinkisJobOperator oper; - try { - oper = (LinkisJobOperator) JobOperatorFactory.getReusable(AppKeys.REUSABLE_UJES_CLIENT); - } catch (Exception e) { - throw new LinkisClientRuntimeException( - "BLD0012", - ErrorLevel.ERROR, - CommonErrMsg.BuilderBuildErr, - "Failed to get a valid operator.", - e); - } - return oper; - } - - @Override - protected PresentWay buildPresentWay() { - PresentWayImpl presentWay = new PresentWayImpl(); - String outputPath = stdVarAccess.getVar(String.class, AppKeys.LINKIS_CLIENT_COMMON_OUTPUT_PATH); - - presentWay.setPath(outputPath); - presentWay.setMode(PresentModeImpl.STDOUT); - if (StringUtils.isNotBlank(outputPath)) { - presentWay.setMode(PresentModeImpl.TEXT_FILE); - } - - return presentWay; - } - - @Override - protected LinkisManageJob getTargetNewInstance() { - return new LinkisManageJob(); - } - - @Override - public LinkisManageJob build() { - ((LinkisManageJob) targetObj).setJobDesc(buildJobDesc()); - ((LinkisManageJob) targetObj).setJobData(buildJobData()); - targetObj.setOperator(buildJobOperator()); - targetObj.setPresentWay(buildPresentWay()); - return (LinkisManageJob) super.build(); - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/builder/LinkisOnceJobBuilder.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/builder/LinkisOnceJobBuilder.java deleted file mode 100644 index 15a2eb3ac3..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/builder/LinkisOnceJobBuilder.java +++ /dev/null @@ -1,239 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.application.interactor.job.builder; - -import org.apache.linkis.cli.application.constants.AppConstants; -import org.apache.linkis.cli.application.constants.AppKeys; -import org.apache.linkis.cli.application.constants.LinkisKeys; -import org.apache.linkis.cli.application.interactor.job.LinkisOnceJob; -import org.apache.linkis.cli.application.interactor.job.data.LinkisOnceJobData; -import org.apache.linkis.cli.application.interactor.job.data.SimpleOnceJobAdapter; -import org.apache.linkis.cli.application.interactor.job.desc.LinkisOnceDesc; -import org.apache.linkis.cli.application.observer.listener.LinkisClientListener; -import org.apache.linkis.cli.application.utils.ExecutionUtils; -import org.apache.linkis.cli.common.entity.operator.JobOperator; -import org.apache.linkis.cli.common.entity.present.PresentWay; -import org.apache.linkis.cli.core.interactor.job.JobBuilder; -import org.apache.linkis.cli.core.present.PresentModeImpl; -import org.apache.linkis.cli.core.present.PresentWayImpl; - -import org.apache.commons.lang3.StringUtils; - -import java.util.*; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class LinkisOnceJobBuilder extends JobBuilder { - private static Logger logger = LoggerFactory.getLogger(LinkisSubmitJobBuilder.class); - - private LinkisClientListener logListener; - private Boolean isAsync = false; - private SimpleOnceJobAdapter onceJobAdapter = new SimpleOnceJobAdapter(); - - public LinkisOnceJobBuilder setLogListener(LinkisClientListener observer) { - this.logListener = observer; - return this; - } - - public LinkisOnceJobBuilder setAsync(Boolean async) { - isAsync = async; - return this; - } - - @Override - protected LinkisOnceJob getTargetNewInstance() { - return new LinkisOnceJob(); - } - - @Override - protected LinkisOnceDesc buildJobDesc() { - LinkisOnceDesc desc = new LinkisOnceDesc(); - - desc.setStdVarAccess(stdVarAccess); - desc.setSysVarAccess(sysVarAccess); - - Map confMap = stdVarAccess.getVar(Map.class, AppKeys.JOB_PARAM_CONF); - Map runtimeMap = stdVarAccess.getVar(Map.class, AppKeys.JOB_PARAM_RUNTIME); - Map varMap = stdVarAccess.getVar(Map.class, AppKeys.JOB_PARAM_VAR); - Map labelMap = stdVarAccess.getVar(Map.class, AppKeys.JOB_LABEL); - Map sourceMap = stdVarAccess.getVar(Map.class, AppKeys.JOB_SOURCE); - Map executionMap = stdVarAccess.getVar(Map.class, AppKeys.JOB_EXEC); - Map jobContentMap = stdVarAccess.getVar(Map.class, AppKeys.JOB_CONTENT); - - confMap = confMap == null ? new HashMap<>() : confMap; - runtimeMap = runtimeMap == null ? new HashMap<>() : runtimeMap; - varMap = varMap == null ? new HashMap<>() : varMap; - labelMap = labelMap == null ? new HashMap<>() : labelMap; - sourceMap = sourceMap == null ? new HashMap<>() : sourceMap; - executionMap = executionMap == null ? new HashMap<>() : executionMap; - jobContentMap = jobContentMap == null ? new HashMap<>() : jobContentMap; - - confMap = ProcessKeyUtils.removePrefixForKeysInMap(confMap); - runtimeMap = ProcessKeyUtils.removePrefixForKeysInMap(runtimeMap); - labelMap = ProcessKeyUtils.removePrefixForKeysInMap(labelMap); - sourceMap = ProcessKeyUtils.removePrefixForKeysInMap(sourceMap); - executionMap = ProcessKeyUtils.removePrefixForKeysInMap(executionMap); - jobContentMap = ProcessKeyUtils.removePrefixForKeysInMap(jobContentMap); - - for (String key : stdVarAccess.getAllVarKeys()) { - Object val = stdVarAccess.getVar(Object.class, key); - if (!(val instanceof Map) && val != null) { - // note that we allow it to overwrite existing values in map - if (StringUtils.startsWithIgnoreCase(key, AppKeys.JOB_PARAM_CONF)) { - ProcessKeyUtils.removePrefixAndPutValToMap(confMap, key, val, AppKeys.JOB_PARAM_CONF); - } else if (StringUtils.startsWithIgnoreCase(key, AppKeys.JOB_PARAM_VAR)) { - ProcessKeyUtils.removePrefixAndPutValToMap(varMap, key, val, AppKeys.JOB_PARAM_VAR); - } else if (StringUtils.startsWithIgnoreCase(key, AppKeys.JOB_PARAM_RUNTIME)) { - ProcessKeyUtils.removePrefixAndPutValToMap( - runtimeMap, key, val, AppKeys.JOB_PARAM_RUNTIME); - } else if (StringUtils.startsWithIgnoreCase(key, AppKeys.JOB_EXEC)) { - ProcessKeyUtils.removePrefixAndPutValToMap(executionMap, key, val, AppKeys.JOB_EXEC); - } else if (StringUtils.startsWithIgnoreCase(key, AppKeys.JOB_LABEL)) { - ProcessKeyUtils.removePrefixAndPutValToMap(labelMap, key, val, AppKeys.JOB_LABEL); - } else if (StringUtils.startsWithIgnoreCase(key, AppKeys.JOB_SOURCE)) { - ProcessKeyUtils.removePrefixAndPutValToMap(sourceMap, key, val, AppKeys.JOB_SOURCE); - } else if (StringUtils.startsWithIgnoreCase(key, AppKeys.JOB_CONTENT)) { - ProcessKeyUtils.removePrefixAndPutValToMap(jobContentMap, key, val, AppKeys.JOB_CONTENT); - } else if (StringUtils.startsWithIgnoreCase(key, AppKeys.LINKIS_CLIENT_COMMON)) { - // do nothing - } else { - // confMap.put(key, stdVarAccess.getVar(Object.class, key)); - } - } - } - - String creator; - if (!isAsync) { - creator = - stdVarAccess.getVarOrDefault( - String.class, AppKeys.JOB_COMMON_CREATOR, AppConstants.JOB_CREATOR_DEFAULT); - } else { - creator = - stdVarAccess.getVarOrDefault( - String.class, AppKeys.JOB_COMMON_CREATOR, AppConstants.JOB_CREATOR_ASYNC_DEFAULT); - } - String code = stdVarAccess.getVar(String.class, AppKeys.JOB_EXEC_CODE); - String engineType = stdVarAccess.getVar(String.class, AppKeys.JOB_LABEL_ENGINE_TYPE); - String runType = stdVarAccess.getVar(String.class, AppKeys.JOB_LABEL_CODE_TYPE); - String scriptPath = - stdVarAccess.getVarOrDefault(String.class, AppKeys.JOB_SOURCE_SCRIPT_PATH, "LinkisCli"); - - String osUser = sysVarAccess.getVar(String.class, AppKeys.LINUX_USER_KEY); - String[] adminUsers = StringUtils.split(AppKeys.ADMIN_USERS, ','); - Set adminSet = new HashSet<>(); - for (String admin : adminUsers) { - adminSet.add(admin); - } - String submitUsr = ExecutionUtils.getSubmitUser(stdVarAccess, osUser, adminSet); - String proxyUsr = ExecutionUtils.getProxyUser(stdVarAccess, submitUsr, adminSet); - - String enableExecuteOnce = - stdVarAccess.getVarOrDefault(String.class, AppKeys.JOB_LABEL_EXECUTEONCE, "true"); - // default executeOnce-mode - if (Boolean.parseBoolean(enableExecuteOnce)) { - labelMap.put(LinkisKeys.KEY_EXECUTEONCE, ""); - } else { - labelMap.remove(LinkisKeys.KEY_EXECUTEONCE); - } - String codePath = stdVarAccess.getVar(String.class, AppKeys.JOB_COMMON_CODE_PATH); - Object extraArgsObj = stdVarAccess.getVar(Object.class, AppKeys.JOB_EXTRA_ARGUMENTS); - if (extraArgsObj != null - && extraArgsObj instanceof String[] - && StringUtils.isBlank(code) - && StringUtils.isBlank(codePath)) { - String[] extraArgs = (String[]) extraArgsObj; - codePath = extraArgs[0]; - if (extraArgs.length > 1) { - runtimeMap.put( - LinkisKeys.EXTRA_ARGUMENTS, Arrays.copyOfRange(extraArgs, 1, extraArgs.length)); - } - } - - if (StringUtils.isBlank(code) && StringUtils.isNotBlank(codePath)) { - code = ExecutionUtils.readFile(codePath); - } - - executionMap.put(LinkisKeys.KEY_CODE, code); - labelMap.put(LinkisKeys.KEY_ENGINETYPE, engineType); - labelMap.put(LinkisKeys.KEY_CODETYPE, runType); - labelMap.put(LinkisKeys.KEY_USER_CREATOR, proxyUsr + "-" + creator); - sourceMap.put(LinkisKeys.KEY_SCRIPT_PATH, scriptPath); - runtimeMap.put(LinkisKeys.KEY_HIVE_RESULT_DISPLAY_TBALE, true); - - desc.setCreator(creator); - desc.setParamConfMap(confMap); - desc.setParamRunTimeMap(runtimeMap); - desc.setParamVarsMap(varMap); - desc.setLabelMap(labelMap); - desc.setSourceMap(sourceMap); - desc.setExecutionMap(executionMap); - desc.setSubmitUser(submitUsr); - desc.setProxyUser(proxyUsr); - desc.setJobContentMap(jobContentMap); - - return desc; - } - - @Override - protected LinkisOnceJobData buildJobData() { - LinkisOnceJobData data = new LinkisOnceJobData(); - data.setOnceJobAdapter(this.onceJobAdapter); - if (logListener == null) { - logger.warn("logListener is not registered, will not be able to display log"); - } else { - data.registerincLogListener(logListener); - } - return data; - } - - @Override - protected JobOperator buildJobOperator() { - // OnceJob is Stateful, should not have an operator - return null; - } - - @Override - protected PresentWay buildPresentWay() { - PresentWayImpl presentWay = new PresentWayImpl(); - String outputPath = stdVarAccess.getVar(String.class, AppKeys.LINKIS_CLIENT_COMMON_OUTPUT_PATH); - - presentWay.setPath(outputPath); - presentWay.setMode(PresentModeImpl.STDOUT); - presentWay.setDisplayMetaAndLogo( - stdVarAccess.getVarOrDefault(Boolean.class, AppKeys.LINKIS_COMMON_DIAPLAY_META_LOGO, true)); - if (StringUtils.isNotBlank(outputPath)) { - presentWay.setMode(PresentModeImpl.TEXT_FILE); - } - - return presentWay; - } - - @Override - public LinkisOnceJob build() { - LinkisOnceDesc desc = buildJobDesc(); - ((LinkisOnceJob) targetObj).setJobDesc(desc); - LinkisOnceJobData data = buildJobData(); - ((LinkisOnceJob) targetObj).setJobData(data); - data.getOnceJobAdapter().init(desc); - ((LinkisOnceJob) targetObj).setAsync(isAsync); - targetObj.setOperator(buildJobOperator()); - targetObj.setPresentWay(buildPresentWay()); - return (LinkisOnceJob) super.build(); - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/builder/LinkisSubmitJobBuilder.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/builder/LinkisSubmitJobBuilder.java deleted file mode 100644 index 6804f09056..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/builder/LinkisSubmitJobBuilder.java +++ /dev/null @@ -1,241 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.application.interactor.job.builder; - -import org.apache.linkis.cli.application.constants.AppConstants; -import org.apache.linkis.cli.application.constants.AppKeys; -import org.apache.linkis.cli.application.constants.LinkisKeys; -import org.apache.linkis.cli.application.interactor.job.LinkisSubmitJob; -import org.apache.linkis.cli.application.interactor.job.data.LinkisJobData; -import org.apache.linkis.cli.application.interactor.job.data.LinkisJobDataImpl; -import org.apache.linkis.cli.application.interactor.job.desc.LinkisSubmitDesc; -import org.apache.linkis.cli.application.observer.listener.LinkisClientListener; -import org.apache.linkis.cli.application.operator.ujes.LinkisJobOperator; -import org.apache.linkis.cli.application.utils.ExecutionUtils; -import org.apache.linkis.cli.common.entity.present.PresentWay; -import org.apache.linkis.cli.common.exception.LinkisClientRuntimeException; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; -import org.apache.linkis.cli.core.interactor.job.JobBuilder; -import org.apache.linkis.cli.core.operator.JobOperatorFactory; -import org.apache.linkis.cli.core.present.PresentModeImpl; -import org.apache.linkis.cli.core.present.PresentWayImpl; - -import org.apache.commons.lang3.StringUtils; - -import java.util.*; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class LinkisSubmitJobBuilder extends JobBuilder { - private static Logger logger = LoggerFactory.getLogger(LinkisSubmitJobBuilder.class); - - private LinkisClientListener logListener; - private Boolean isAsync = false; - - public LinkisSubmitJobBuilder setLogListener(LinkisClientListener observer) { - this.logListener = observer; - return this; - } - - public LinkisSubmitJobBuilder setAsync(Boolean async) { - isAsync = async; - return this; - } - - @Override - protected LinkisSubmitJob getTargetNewInstance() { - return new LinkisSubmitJob(); - } - - @Override - protected LinkisSubmitDesc buildJobDesc() { - LinkisSubmitDesc desc = new LinkisSubmitDesc(); - - Map confMap = stdVarAccess.getVar(Map.class, AppKeys.JOB_PARAM_CONF); - Map runtimeMap = stdVarAccess.getVar(Map.class, AppKeys.JOB_PARAM_RUNTIME); - Map varMap = stdVarAccess.getVar(Map.class, AppKeys.JOB_PARAM_VAR); - Map labelMap = stdVarAccess.getVar(Map.class, AppKeys.JOB_LABEL); - Map sourceMap = stdVarAccess.getVar(Map.class, AppKeys.JOB_SOURCE); - Map executionMap = stdVarAccess.getVar(Map.class, AppKeys.JOB_EXEC); - - confMap = confMap == null ? new HashMap<>() : confMap; - runtimeMap = runtimeMap == null ? new HashMap<>() : runtimeMap; - varMap = varMap == null ? new HashMap<>() : varMap; - labelMap = labelMap == null ? new HashMap<>() : labelMap; - sourceMap = sourceMap == null ? new HashMap<>() : sourceMap; - executionMap = executionMap == null ? new HashMap<>() : executionMap; - - /** remove key prefix of all keys in map type params. e.g. kv in confMap, labelMap etc. */ - confMap = ProcessKeyUtils.removePrefixForKeysInMap(confMap); - runtimeMap = ProcessKeyUtils.removePrefixForKeysInMap(runtimeMap); - labelMap = ProcessKeyUtils.removePrefixForKeysInMap(labelMap); - sourceMap = ProcessKeyUtils.removePrefixForKeysInMap(sourceMap); - executionMap = ProcessKeyUtils.removePrefixForKeysInMap(executionMap); - - /** remove key prefix of non-map type params */ - for (String key : stdVarAccess.getAllVarKeys()) { - Object val = stdVarAccess.getVar(Object.class, key); - if (!(val instanceof Map) && val != null) { - // note that we allow it to overwrite existing values in map - if (StringUtils.startsWithIgnoreCase(key, AppKeys.JOB_PARAM_CONF)) { - ProcessKeyUtils.removePrefixAndPutValToMap(confMap, key, val, AppKeys.JOB_PARAM_CONF); - } else if (StringUtils.startsWithIgnoreCase(key, AppKeys.JOB_PARAM_VAR)) { - ProcessKeyUtils.removePrefixAndPutValToMap(varMap, key, val, AppKeys.JOB_PARAM_VAR); - } else if (StringUtils.startsWithIgnoreCase(key, AppKeys.JOB_PARAM_RUNTIME)) { - ProcessKeyUtils.removePrefixAndPutValToMap( - runtimeMap, key, val, AppKeys.JOB_PARAM_RUNTIME); - } else if (StringUtils.startsWithIgnoreCase(key, AppKeys.JOB_EXEC)) { - ProcessKeyUtils.removePrefixAndPutValToMap(executionMap, key, val, AppKeys.JOB_EXEC); - } else if (StringUtils.startsWithIgnoreCase(key, AppKeys.JOB_LABEL)) { - ProcessKeyUtils.removePrefixAndPutValToMap(labelMap, key, val, AppKeys.JOB_LABEL); - } else if (StringUtils.startsWithIgnoreCase(key, AppKeys.JOB_SOURCE)) { - ProcessKeyUtils.removePrefixAndPutValToMap(sourceMap, key, val, AppKeys.JOB_SOURCE); - } else if (StringUtils.startsWithIgnoreCase(key, AppKeys.LINKIS_CLIENT_COMMON)) { - // do nothing - } else { - // confMap.put(key, stdVarAccess.getVar(Object.class, key)); - } - } - } - - String creator; - if (!isAsync) { - creator = - stdVarAccess.getVarOrDefault( - String.class, AppKeys.JOB_COMMON_CREATOR, AppConstants.JOB_CREATOR_DEFAULT); - } else { - creator = - stdVarAccess.getVarOrDefault( - String.class, AppKeys.JOB_COMMON_CREATOR, AppConstants.JOB_CREATOR_ASYNC_DEFAULT); - } - String code = stdVarAccess.getVar(String.class, AppKeys.JOB_EXEC_CODE); - String engineType = stdVarAccess.getVar(String.class, AppKeys.JOB_LABEL_ENGINE_TYPE); - String runType = stdVarAccess.getVar(String.class, AppKeys.JOB_LABEL_CODE_TYPE); - String scriptPath = - stdVarAccess.getVarOrDefault(String.class, AppKeys.JOB_SOURCE_SCRIPT_PATH, "LinkisCli"); - - String osUser = sysVarAccess.getVar(String.class, AppKeys.LINUX_USER_KEY); - String[] adminUsers = StringUtils.split(AppKeys.ADMIN_USERS, ','); - Set adminSet = new HashSet<>(); - for (String admin : adminUsers) { - adminSet.add(admin); - } - String submitUsr = ExecutionUtils.getSubmitUser(stdVarAccess, osUser, adminSet); - String proxyUsr = ExecutionUtils.getProxyUser(stdVarAccess, submitUsr, adminSet); - - String enableExecuteOnce = - stdVarAccess.getVarOrDefault(String.class, AppKeys.JOB_LABEL_EXECUTEONCE, "true"); - // default executeOnce-mode - if (Boolean.parseBoolean(enableExecuteOnce)) { - labelMap.put(LinkisKeys.KEY_EXECUTEONCE, ""); - } else { - labelMap.remove(LinkisKeys.KEY_EXECUTEONCE); - } - String codePath = stdVarAccess.getVar(String.class, AppKeys.JOB_COMMON_CODE_PATH); - Object extraArgsObj = stdVarAccess.getVar(Object.class, AppKeys.JOB_EXTRA_ARGUMENTS); - if (extraArgsObj != null - && extraArgsObj instanceof String[] - && StringUtils.isBlank(code) - && StringUtils.isBlank(codePath)) { - String[] extraArgs = (String[]) extraArgsObj; - codePath = extraArgs[0]; - if (extraArgs.length > 1) { - runtimeMap.put( - LinkisKeys.EXTRA_ARGUMENTS, Arrays.copyOfRange(extraArgs, 1, extraArgs.length)); - } - } - - if (StringUtils.isBlank(code) && StringUtils.isNotBlank(codePath)) { - code = ExecutionUtils.readFile(codePath); - } - - executionMap.put(LinkisKeys.KEY_CODE, code); - labelMap.put(LinkisKeys.KEY_ENGINETYPE, engineType); - labelMap.put(LinkisKeys.KEY_CODETYPE, runType); - labelMap.put(LinkisKeys.KEY_USER_CREATOR, proxyUsr + "-" + creator); - sourceMap.put(LinkisKeys.KEY_SCRIPT_PATH, scriptPath); - runtimeMap.put(LinkisKeys.KEY_HIVE_RESULT_DISPLAY_TBALE, true); - - desc.setCreator(creator); - desc.setParamConfMap(confMap); - desc.setParamRunTimeMap(runtimeMap); - desc.setParamVarsMap(varMap); - desc.setLabelMap(labelMap); - desc.setSourceMap(sourceMap); - desc.setExecutionMap(executionMap); - desc.setSubmitUser(submitUsr); - desc.setProxyUser(proxyUsr); - - return desc; - } - - @Override - protected LinkisJobData buildJobData() { - LinkisJobDataImpl data = new LinkisJobDataImpl(); - if (logListener == null) { - logger.warn("logListener is not registered, will not be able to display log"); - } else { - data.registerincLogListener(logListener); - } - return data; - } - - @Override - protected LinkisJobOperator buildJobOperator() { - LinkisJobOperator oper; - try { - oper = (LinkisJobOperator) JobOperatorFactory.getReusable(AppKeys.REUSABLE_UJES_CLIENT); - } catch (Exception e) { - throw new LinkisClientRuntimeException( - "BLD0012", - ErrorLevel.ERROR, - CommonErrMsg.BuilderBuildErr, - "Failed to get a valid operator.", - e); - } - return oper; - } - - @Override - protected PresentWay buildPresentWay() { - PresentWayImpl presentWay = new PresentWayImpl(); - String outputPath = stdVarAccess.getVar(String.class, AppKeys.LINKIS_CLIENT_COMMON_OUTPUT_PATH); - - presentWay.setPath(outputPath); - presentWay.setMode(PresentModeImpl.STDOUT); - presentWay.setDisplayMetaAndLogo( - stdVarAccess.getVarOrDefault(Boolean.class, AppKeys.LINKIS_COMMON_DIAPLAY_META_LOGO, true)); - if (StringUtils.isNotBlank(outputPath)) { - presentWay.setMode(PresentModeImpl.TEXT_FILE); - } - - return presentWay; - } - - @Override - public LinkisSubmitJob build() { - ((LinkisSubmitJob) targetObj).setJobDesc(buildJobDesc()); - ((LinkisSubmitJob) targetObj).setJobData(buildJobData()); - ((LinkisSubmitJob) targetObj).setAsync(isAsync); - targetObj.setOperator(buildJobOperator()); - targetObj.setPresentWay(buildPresentWay()); - return (LinkisSubmitJob) super.build(); - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/data/LinkisJobDataImpl.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/data/LinkisJobDataImpl.java deleted file mode 100644 index 227e8c2170..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/data/LinkisJobDataImpl.java +++ /dev/null @@ -1,514 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.application.interactor.job.data; - -import org.apache.linkis.cli.application.observer.event.LinkisClientEvent; -import org.apache.linkis.cli.application.observer.event.LogStartEvent; -import org.apache.linkis.cli.application.observer.event.TriggerEvent; -import org.apache.linkis.cli.application.observer.listener.LinkisClientListener; -import org.apache.linkis.cli.application.observer.listener.TriggerEventListener; -import org.apache.linkis.cli.application.operator.ujes.LinkisOperResultAdapter; -import org.apache.linkis.cli.common.entity.job.JobStatus; - -import java.util.Arrays; -import java.util.Date; -import java.util.LinkedList; -import java.util.List; -import java.util.concurrent.LinkedBlockingDeque; - -public class LinkisJobDataImpl - implements LinkisJobData, LinkisLogData, LinkisResultData, Cloneable { - - private String jobID; - private String user; - private JobStatus jobStatus = null; - private String message; - private Exception exception; - private String execID; - private float progress = 0.0f; - private Boolean incLogMode; - private LinkedBlockingDeque logBuffer = new LinkedBlockingDeque(); - private String logPath; // remote path for job log - private Integer nextLogLineIdx; - private Boolean hasNextLogLine; - private String resultLocation; - private String[] resultSetPaths = null; // remote paths for job result set - private LinkedBlockingDeque resultContent = new LinkedBlockingDeque<>(); - private Boolean hasNextResultPage; - private Integer errCode = null; - private String errDesc = null; - private boolean success = false; - private String instance; - private String umUser; - private String simpleExecId; - private String executionCode; - private String engineType; - private String runType; - private Long costTime; - private Date createdTime; - private Date updatedTime; - private Date engineStartTime; - private String executeApplicationName; - private String requestApplicationName; - - private LinkisClientEvent logstartEvent = new LogStartEvent(); - private TriggerEvent logFinevent = new TriggerEvent(); - private TriggerEventListener logFinListener = new TriggerEventListener(); - private TriggerEvent resultFinEvent = new TriggerEvent(); - private TriggerEventListener resultFinListener = new TriggerEventListener(); - - private boolean hasResult = true; - - { - logFinevent.register(logFinListener); - resultFinEvent.register(resultFinListener); - } - - @Override - public String getJobID() { - return jobID; - } - - public void setJobId(String jobId) { - this.jobID = jobId; - } - - @Override - public String getUser() { - return user; - } - - public void setUser(String user) { - this.user = user; - } - - @Override - public JobStatus getJobStatus() { - return jobStatus; - } - - public void setJobStatus(JobStatus jobStatus) { - this.jobStatus = jobStatus; - } - - @Override - public String getMessage() { - return message; - } - - @Override - public void setMessage(String message) { - this.message = message; - } - - @Override - public Exception getException() { - return exception; - } - - @Override - public void setException(Exception exception) { - this.exception = exception; - } - - @Override - public final String getExecID() { - return execID; - } - - public final void setExecID(String execID) { - this.execID = execID; - } - - @Override - public final float getJobProgress() { - return progress; - } - - public final void setJobProgress(float progress) { - this.progress = progress; - } - - @Override - public final String getLogPath() { - return logPath; - } - - public final void setLogPath(String logPath) { - this.logPath = logPath; - } - - @Override - public final String getResultLocation() { - return resultLocation; - } - - public final void setResultLocation(String resultLocation) { - this.resultLocation = resultLocation; - } - - @Override - public String[] getResultSetPaths() { - return resultSetPaths; - } - - public final void setResultSetPaths(String[] resultSetPaths) { - this.resultSetPaths = resultSetPaths; - } - - @Override - public Integer getErrCode() { - return errCode; - } - - public void setErrCode(Integer errCode) { - this.errCode = errCode; - } - - @Override - public String getErrDesc() { - return errDesc; - } - - public void setErrDesc(String errDesc) { - this.errDesc = errDesc; - } - - @Override - public void registerincLogListener(LinkisClientListener observer) { - this.logstartEvent.register(observer); - } - - @Override - public void notifyLogListener() { - if (this.logstartEvent.isRegistered()) { - logstartEvent.notifyObserver(logstartEvent, this); - } - } - - @Override - public boolean isIncLogMode() { - return this.incLogMode; - } - - @Override - public void setIncLogMode(boolean incLogMode) { - this.incLogMode = incLogMode; - } - - @Override - public String consumeLog() { - List logs = new LinkedList<>(); - this.logBuffer.drainTo(logs, this.logBuffer.size()); - StringBuilder tmp = new StringBuilder(); - for (String str : logs) { - tmp.append(str); - } - return tmp.toString(); - } - - public void appendLog(String log) { - this.logBuffer.add(log); - } - - @Override - public Integer getNextLogLineIdx() { - return nextLogLineIdx; - } - - public void setNextLogLineIdx(Integer nextLogLineIdx) { - this.nextLogLineIdx = nextLogLineIdx; - } - - @Override - public Boolean hasNextLogLine() { - return hasNextLogLine; - } - - @Override - public void setHasNextLogLine(Boolean hasNextLogLine) { - this.hasNextLogLine = hasNextLogLine; - } - - @Override - public List consumeResultContent() { - List ret = new LinkedList<>(); - resultContent.drainTo(ret, resultContent.size()); - return ret; - } - - public void appendResultContent(LinkisResultSet resultContent) { - this.resultContent.add(resultContent); - } - - @Override - public Boolean hasNextResultPage() { - return hasNextResultPage; - } - - public void setHasNextResultPage(Boolean hasNextResultPage) { - this.hasNextResultPage = hasNextResultPage; - } - - @Override - public void sendLogFin() { - if (this.logFinevent != null && this.logFinevent.isRegistered()) { - this.logFinevent.notifyObserver(resultFinEvent, null); - } - } - - @Override - public boolean logFinReceived() { - return this.logFinListener.isTriggered(); - } - - @Override - public void sendResultFin() { - if (this.resultFinEvent != null && this.resultFinEvent.isRegistered()) { - this.resultFinEvent.notifyObserver(resultFinEvent, null); - } - } - - @Override - public boolean resultFinReceived() { - return this.resultFinListener.isTriggered(); - } - - @Override - public boolean hasResult() { - return hasResult; - } - - @Override - public void setHasResult(boolean hasResult) { - this.hasResult = hasResult; - } - - @Override - public boolean isSuccess() { - return success; - } - - @Override - public void setSuccess(boolean success) { - this.success = success; - } - - public String getInstance() { - return instance; - } - - public void setInstance(String instance) { - this.instance = instance; - } - - public String getUmUser() { - return umUser; - } - - public void setUmUser(String umUser) { - this.umUser = umUser; - } - - public String getSimpleExecId() { - return simpleExecId; - } - - public void setSimpleExecId(String simpleExecId) { - this.simpleExecId = simpleExecId; - } - - public String getExecutionCode() { - return executionCode; - } - - public void setExecutionCode(String executionCode) { - this.executionCode = executionCode; - } - - public String getEngineType() { - return engineType; - } - - public void setEngineType(String engineType) { - this.engineType = engineType; - } - - public String getRunType() { - return runType; - } - - public void setRunType(String runType) { - this.runType = runType; - } - - public Long getCostTime() { - return costTime; - } - - public void setCostTime(Long costTime) { - this.costTime = costTime; - } - - public Date getCreatedTime() { - return createdTime; - } - - public void setCreatedTime(Date createdTime) { - this.createdTime = createdTime; - } - - public Date getUpdatedTime() { - return updatedTime; - } - - public void setUpdatedTime(Date updatedTime) { - this.updatedTime = updatedTime; - } - - public Date getEngineStartTime() { - return engineStartTime; - } - - public void setEngineStartTime(Date engineStartTime) { - this.engineStartTime = engineStartTime; - } - - public String getExecuteApplicationName() { - return executeApplicationName; - } - - public void setExecuteApplicationName(String executeApplicationName) { - this.executeApplicationName = executeApplicationName; - } - - public String getRequestApplicationName() { - return requestApplicationName; - } - - public void setRequestApplicationName(String requestApplicationName) { - this.requestApplicationName = requestApplicationName; - } - - @Override - public void updateByOperResult(LinkisOperResultAdapter adapter) { - if (adapter.getJobID() != null) { - setJobId(adapter.getJobID()); - } - if (adapter.getUser() != null) { - setUser(adapter.getUser()); - } - if (adapter.getJobStatus() != null) { - setJobStatus(adapter.getJobStatus()); - } - if (adapter.getStrongerExecId() != null) { - setExecID(adapter.getStrongerExecId()); - } - if (adapter.getJobProgress() != null) { - setJobProgress(adapter.getJobProgress()); - } - if (adapter.getLogPath() != null) { - setLogPath(adapter.getLogPath()); - } - if (adapter.getResultLocation() != null) { - setResultLocation(adapter.getResultLocation()); - } - if (adapter.getResultSetPaths() != null) { - setResultSetPaths(adapter.getResultSetPaths()); - } - if (adapter.getErrCode() != null) { - setErrCode(adapter.getErrCode()); - } - if (adapter.getErrDesc() != null) { - setErrDesc(adapter.getErrDesc()); - } - if (adapter.getLog() != null - && adapter.getNextLogLine() != null - && adapter.hasNextLogLine() != null) { - setNextLogLineIdx(adapter.getNextLogLine()); - setHasNextLogLine(adapter.hasNextLogLine()); - appendLog(adapter.getLog()); - } - if (adapter.getResultContent() != null && adapter.resultHasNextPage() != null) { - setHasNextResultPage(adapter.resultHasNextPage()); - appendResultContent(adapter.getResultContent()); - } - if (adapter.getInstance() != null) { - setInstance(adapter.getInstance()); - } - if (adapter.getUmUser() != null) { - setUmUser(adapter.getUmUser()); - } - if (adapter.getSimpleExecId() != null) { - setSimpleExecId(adapter.getSimpleExecId()); - } - if (adapter.getExecutionCode() != null) { - setExecutionCode(adapter.getExecutionCode()); - } - if (adapter.getEngineType() != null) { - setEngineType(adapter.getEngineType()); - } - if (adapter.getRunType() != null) { - setRunType(adapter.getRunType()); - } - if (adapter.getCostTime() != null) { - setCostTime(adapter.getCostTime()); - } - if (adapter.getCreatedTime() != null) { - setCreatedTime(adapter.getCreatedTime()); - } - if (adapter.getUpdatedTime() != null) { - setUpdatedTime(adapter.getUpdatedTime()); - } - if (adapter.getEngineStartTime() != null) { - setEngineStartTime(adapter.getEngineStartTime()); - } - if (adapter.getExecuteApplicationName() != null) { - setExecuteApplicationName(adapter.getExecuteApplicationName()); - } - if (adapter.getRequestApplicationName() != null) { - setRequestApplicationName(adapter.getRequestApplicationName()); - } - } - - @Override - public LinkisJobDataImpl clone() throws CloneNotSupportedException { - LinkisJobDataImpl ret = (LinkisJobDataImpl) super.clone(); - if (logBuffer != null) { - ret.logBuffer = new LinkedBlockingDeque(this.logBuffer); - } - if (this.resultContent != null) { - ret.resultContent = new LinkedBlockingDeque<>(); - for (LinkisResultSet r1 : resultContent) { - ret.resultContent.add(r1.clone()); - } - } - if (this.resultSetPaths != null) { - ret.setResultSetPaths(Arrays.copyOf(this.resultSetPaths, this.resultSetPaths.length)); - } - /* - These be shared and hence should not be deep copied. - */ - ret.logFinevent = this.logFinevent; - ret.logFinListener = this.logFinListener; - ret.resultFinEvent = this.resultFinEvent; - ret.resultFinListener = this.resultFinListener; - - return ret; - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/data/LinkisLogData.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/data/LinkisLogData.java deleted file mode 100644 index 3df7cc5a47..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/data/LinkisLogData.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.application.interactor.job.data; - -import org.apache.linkis.cli.application.observer.listener.LinkisClientListener; - -public interface LinkisLogData extends LinkisJobData, Cloneable { - - void notifyLogListener(); - - void registerincLogListener(LinkisClientListener observer); - - String getLogPath(); - - Integer getNextLogLineIdx(); - - void setHasNextLogLine(Boolean hasNextLog); - - Boolean hasNextLogLine(); - - String consumeLog(); - - boolean isIncLogMode(); - - /* - incLogMode = true: for sync-submission, wait for job complete while get incremental log - incLogMode = false: for async-submission, output all log we have currently - */ - void setIncLogMode(boolean incLogMode); - - void sendLogFin(); - - boolean logFinReceived(); - - LinkisLogData clone() throws CloneNotSupportedException; -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/data/LinkisOnceJobData.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/data/LinkisOnceJobData.java deleted file mode 100644 index 89130b62af..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/data/LinkisOnceJobData.java +++ /dev/null @@ -1,242 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.application.interactor.job.data; - -import org.apache.linkis.cli.application.observer.listener.LinkisClientListener; -import org.apache.linkis.cli.application.operator.ujes.LinkisOperResultAdapter; -import org.apache.linkis.cli.common.entity.job.JobStatus; - -import java.util.List; - -public class LinkisOnceJobData implements LinkisJobData, LinkisLogData, LinkisResultData { - - private SimpleOnceJobAdapter onceJobAdapter; - - private boolean hasResult = true; - - public SimpleOnceJobAdapter getOnceJobAdapter() { - return onceJobAdapter; - } - - public void setOnceJobAdapter(SimpleOnceJobAdapter onceJobAdapter) { - this.onceJobAdapter = onceJobAdapter; - } - - @Override - public void registerincLogListener(LinkisClientListener observer) { - onceJobAdapter.registerincLogListener(observer); - } - - @Override - public void notifyLogListener() { - onceJobAdapter.notifyLogListener(); - } - - @Override - public boolean isIncLogMode() { - return onceJobAdapter.isIncLogMode(); - } - - @Override - public void setIncLogMode(boolean incLogMode) { - onceJobAdapter.setIncLogMode(incLogMode); - } - - @Override - public String consumeLog() { - return onceJobAdapter.consumeLog(); - } - - public void appendLog(String log) { - onceJobAdapter.appendLog(log); - } - - @Override - public final String getLogPath() { - return onceJobAdapter.getLogPath(); - } - - public final void setLogPath(String logPath) { - onceJobAdapter.setLogPath(logPath); - } - - @Override - public Integer getNextLogLineIdx() { - return onceJobAdapter.getNextLogLineIdx(); - } - - public void setNextLogLineIdx(Integer nextLogLineIdx) { - onceJobAdapter.setNextLogLineIdx(nextLogLineIdx); - } - - @Override - public Boolean hasNextLogLine() { - return onceJobAdapter.hasNextLogLine(); - } - - @Override - public void setHasNextLogLine(Boolean hasNextLogLine) { - onceJobAdapter.setHasNextLogLine(hasNextLogLine); - } - - @Override - public List consumeResultContent() { - return onceJobAdapter.consumeResultContent(); - } - - public void appendResultContent(LinkisResultSet resultContent) { - onceJobAdapter.appendResultContent(resultContent); - } - - @Override - public Boolean hasNextResultPage() { - return onceJobAdapter.hasNextResultPage(); - } - - public void setHasNextResultPage(Boolean hasNextResultPage) { - onceJobAdapter.setHasNextResultPage(hasNextResultPage); - } - - @Override - public final String getResultLocation() { - return onceJobAdapter.getResultLocation(); - } - - public final void setResultLocation(String resultLocation) { - onceJobAdapter.setResultLocation(resultLocation); - } - - @Override - public String[] getResultSetPaths() { - return onceJobAdapter.getResultSetPaths(); - } - - public final void setResultSetPaths(String[] resultSetPaths) { - onceJobAdapter.setResultSetPaths(resultSetPaths); - } - - @Override - public void sendLogFin() { - onceJobAdapter.sendLogFin(); - } - - @Override - public boolean logFinReceived() { - return onceJobAdapter.logFinReceived(); - } - - @Override - public void sendResultFin() { - onceJobAdapter.sendResultFin(); - } - - @Override - public boolean resultFinReceived() { - return onceJobAdapter.resultFinReceived(); - } - - @Override - public boolean hasResult() { - return hasResult; - } - - @Override - public void setHasResult(boolean hasResult) { - this.hasResult = hasResult; - } - - @Override - public JobStatus getJobStatus() { - return onceJobAdapter.getJobStatus(); - } - - public void setJobStatus(JobStatus jobStatus) { - onceJobAdapter.setJobStatus(jobStatus); - } - - @Override - public String getJobID() { - return onceJobAdapter.getJobID(); - } - - @Override - public String getUser() { - return onceJobAdapter.getUser(); - } - - @Override - public String getMessage() { - return onceJobAdapter.getMessage(); - } - - @Override - public void setMessage(String message) { - onceJobAdapter.setMessage(message); - } - - @Override - public Exception getException() { - return onceJobAdapter.getException(); - } - - @Override - public void setException(Exception e) { - onceJobAdapter.setException(e); - } - - @Override - public String getExecID() { - return onceJobAdapter.getJobID(); - } // No Need - - @Override - public float getJobProgress() { - return 0; - } - - @Override - public Integer getErrCode() { - return onceJobAdapter.getErrCode(); - } - - @Override - public String getErrDesc() { - return onceJobAdapter.getErrDesc(); - } - - @Override - public boolean isSuccess() { - return onceJobAdapter.isSuccess(); - } - - @Override - public void setSuccess(boolean success) { - onceJobAdapter.setSuccess(success); - } - - @Override - public void updateByOperResult(LinkisOperResultAdapter adapter) { - // No need - } - - @Override - public LinkisOnceJobData clone() throws CloneNotSupportedException { - throw new CloneNotSupportedException(); - // return null; - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/data/SimpleOnceJobAdapter.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/data/SimpleOnceJobAdapter.java deleted file mode 100644 index 69a76d6240..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/data/SimpleOnceJobAdapter.java +++ /dev/null @@ -1,377 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.application.interactor.job.data; - -import org.apache.linkis.cli.application.constants.AppConstants; -import org.apache.linkis.cli.application.constants.AppKeys; -import org.apache.linkis.cli.application.constants.LinkisKeys; -import org.apache.linkis.cli.application.interactor.job.LinkisJobStatus; -import org.apache.linkis.cli.application.interactor.job.desc.LinkisOnceDesc; -import org.apache.linkis.cli.application.observer.event.LinkisClientEvent; -import org.apache.linkis.cli.application.observer.event.LogStartEvent; -import org.apache.linkis.cli.application.observer.event.TriggerEvent; -import org.apache.linkis.cli.application.observer.listener.LinkisClientListener; -import org.apache.linkis.cli.application.observer.listener.TriggerEventListener; -import org.apache.linkis.cli.application.operator.once.OnceJobConstants; -import org.apache.linkis.cli.application.operator.ujes.LinkisOperResultAdapter; -import org.apache.linkis.cli.application.operator.ujes.UJESClientFactory; -import org.apache.linkis.cli.common.entity.job.JobStatus; -import org.apache.linkis.cli.common.entity.var.VarAccess; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.exception.LinkisClientExecutionException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; -import org.apache.linkis.computation.client.LinkisJobBuilder$; -import org.apache.linkis.computation.client.once.simple.SimpleOnceJob; -import org.apache.linkis.computation.client.once.simple.SimpleOnceJobBuilder; -import org.apache.linkis.computation.client.once.simple.SubmittableSimpleOnceJob; -import org.apache.linkis.computation.client.operator.impl.EngineConnLogOperator; -import org.apache.linkis.computation.client.operator.impl.EngineConnLogs; - -import org.apache.commons.lang3.StringUtils; - -import java.util.LinkedList; -import java.util.List; -import java.util.concurrent.LinkedBlockingDeque; - -public class SimpleOnceJobAdapter implements LinkisLogData { - LinkisJobStatus jobStatus = LinkisJobStatus.UNSUBMITTED; - EngineConnLogOperator logOperator = null; - private String serverUrl; - private SimpleOnceJob onceJob; - private String engineTypeForECM; - private String message; - private Exception exception; - private boolean success; - private Boolean incLogMode; - private LinkedBlockingDeque logBuffer = new LinkedBlockingDeque(); - // private String logPath; // remote path for job log - // private Integer nextLogLineIdx = 0; - private Boolean hasNextLogLine = true; - private String resultLocation; - private String[] resultSetPaths = null; // remote paths for job result set - private LinkedBlockingDeque resultContent = new LinkedBlockingDeque<>(); - private Boolean hasNextResultPage; - private LinkisClientEvent logstartEvent = new LogStartEvent(); - private TriggerEvent logFinEvent = new TriggerEvent(); - private TriggerEventListener logFinListener = new TriggerEventListener(); - private TriggerEvent resultFinEvent = new TriggerEvent(); - private TriggerEventListener resultFinListener = new TriggerEventListener(); - - { - logFinEvent.register(logFinListener); - resultFinEvent.register(resultFinListener); - } - - public void init(LinkisOnceDesc desc) { - VarAccess stdVarAccess = desc.getStdVarAccess(); - VarAccess sysVarAccess = desc.getSysVarAccess(); - - serverUrl = stdVarAccess.getVar(String.class, AppKeys.LINKIS_COMMON_GATEWAY_URL); - - LinkisJobBuilder$.MODULE$.setDefaultClientConfig( - UJESClientFactory.generateDWSClientConfig(stdVarAccess, sysVarAccess)); - LinkisJobBuilder$.MODULE$.setDefaultUJESClient( - UJESClientFactory.getReusable(stdVarAccess, sysVarAccess)); - - String engineTypeRaw = (String) desc.getLabelMap().get(LinkisKeys.KEY_ENGINETYPE); - engineTypeForECM = engineTypeRaw; - - if (StringUtils.isNotBlank(engineTypeRaw)) { - engineTypeForECM = StringUtils.split(engineTypeRaw, "-")[0]; - } else { - engineTypeForECM = ""; - } // TODO: remove parsing and let server side parse engineType - - onceJob = - new SimpleOnceJobBuilder() - .setCreateService(AppConstants.LINKIS_CLI) - .addExecuteUser(desc.getProxyUser()) - .setStartupParams(desc.getParamConfMap()) - .setLabels(desc.getLabelMap()) - .setRuntimeParams(desc.getParamRunTimeMap()) - .setSource(desc.getSourceMap()) - .setVariableMap(desc.getParamVarsMap()) - .setJobContent(desc.getJobContentMap()) - .build(); - } - - public String getServerUrl() { - return serverUrl; - } - - public SimpleOnceJob getOnceJob() { - return onceJob; - } - - public void setOnceJob(SimpleOnceJob onceJob) { - this.onceJob = onceJob; - } - - private void panicIfNull(Object obj) { - if (obj == null) { - throw new LinkisClientExecutionException( - "EXE0040", - ErrorLevel.ERROR, - CommonErrMsg.ExecutionErr, - "Instance of " + obj.getClass().getCanonicalName() + " is null"); - } - } - - public void submit() { - panicIfNull(onceJob); - if (!(onceJob instanceof SubmittableSimpleOnceJob)) { - throw new LinkisClientExecutionException( - "EXE0041", - ErrorLevel.ERROR, - CommonErrMsg.ExecutionErr, - "onceJob is not properly initiated"); - } - ((SubmittableSimpleOnceJob) onceJob).submit(); - } - - public void kill() { - panicIfNull(onceJob); - onceJob.kill(); - } - - public String getJobID() { - return onceJob.getId(); - } - - @Override - public String getUser() { - return "TODO"; - } - - public void updateStatus() { - panicIfNull(onceJob); - String status = onceJob.getStatus(); - panicIfNull(status); - jobStatus = LinkisJobStatus.convertFromNodeStatusString(onceJob.getStatus()); - } - - public LinkisJobStatus getJobStatus() { - return this.jobStatus; - } - - public void setJobStatus(JobStatus jobStatus) { - this.jobStatus = (LinkisJobStatus) jobStatus; - } - - public void waitForComplete() { - panicIfNull(onceJob); - onceJob.waitForCompleted(); - } - - public void queryJobLogOneIteration() { - panicIfNull(onceJob); - updateStatus(); - if (logOperator == null) { - logOperator = - (EngineConnLogOperator) onceJob.getOperator(EngineConnLogOperator.OPERATOR_NAME()); - logOperator.setECMServiceInstance( - ((SubmittableSimpleOnceJob) onceJob).getECMServiceInstance()); - logOperator.setEngineConnType(engineTypeForECM); - // logOperator.setPageSize(OnceJobConstants.MAX_LOG_SIZE_ONCE); - logOperator.setIgnoreKeywords(OnceJobConstants.LOG_IGNORE_KEYWORDS); - } - EngineConnLogs logs = - (EngineConnLogs) logOperator.apply(); // for some reason we have to add type conversion, - // otherwise mvn testCompile fails - StringBuilder logBuilder = new StringBuilder(); - for (String log : logs.logs()) { - logBuilder.append(log).append(System.lineSeparator()); - } - appendLog(logBuilder.toString()); - if ((logs.logs() == null || logs.logs().size() <= 0) && jobStatus.isJobFinishedState()) { - setHasNextLogLine(false); - } - // System.out.println(logs.logs().size()); - } - - public void registerincLogListener(LinkisClientListener observer) { - this.logstartEvent.register(observer); - } - - public void notifyLogListener() { - if (this.logstartEvent.isRegistered()) { - logstartEvent.notifyObserver(logstartEvent, this); - } - } - - public boolean isIncLogMode() { - return this.incLogMode; - } - - public void setIncLogMode(boolean incLogMode) { - this.incLogMode = incLogMode; - } - - public String consumeLog() { - List logs = new LinkedList<>(); - this.logBuffer.drainTo(logs, this.logBuffer.size()); - StringBuilder tmp = new StringBuilder(); - for (String str : logs) { - tmp.append(str); - } - return tmp.toString(); - } - - public void appendLog(String log) { - this.logBuffer.add(log); - } - - public final String getLogPath() { - return null; - } - - public final void setLogPath(String logPath) { - return; - } - - public Integer getNextLogLineIdx() { - return null; - } - - public void setNextLogLineIdx(Integer nextLogLineIdx) { - return; - } - - public Boolean hasNextLogLine() { - return hasNextLogLine; - } - - public void setHasNextLogLine(Boolean hasNextLogLine) { - this.hasNextLogLine = hasNextLogLine; - } - - public List consumeResultContent() { - List ret = new LinkedList<>(); - resultContent.drainTo(ret, resultContent.size()); - return ret; - } - - public void appendResultContent(LinkisResultSet resultContent) { - this.resultContent.add(resultContent); - } - - public Boolean hasNextResultPage() { - return hasNextResultPage; - } - - public void setHasNextResultPage(Boolean hasNextResultPage) { - this.hasNextResultPage = hasNextResultPage; - } - - public final String getResultLocation() { - return resultLocation; - } - - public final void setResultLocation(String resultLocation) { - this.resultLocation = resultLocation; - } - - public String[] getResultSetPaths() { - return resultSetPaths; - } - - public final void setResultSetPaths(String[] resultSetPaths) { - this.resultSetPaths = resultSetPaths; - } - - public void sendLogFin() { - if (this.logFinEvent != null && this.logFinEvent.isRegistered()) { - this.logFinEvent.notifyObserver(resultFinEvent, null); - } - } - - public boolean logFinReceived() { - return this.logFinListener.isTriggered(); - } - - public void sendResultFin() { - if (this.resultFinEvent != null && this.resultFinEvent.isRegistered()) { - this.resultFinEvent.notifyObserver(resultFinEvent, null); - } - } - - public boolean resultFinReceived() { - return this.resultFinListener.isTriggered(); - } - - @Override - public String getMessage() { - return message; - } - - @Override - public void setMessage(String message) { - this.message = message; - } - - @Override - public Exception getException() { - return exception; - } - - @Override - public void setException(Exception e) { - this.exception = e; - } - - @Override - public String getExecID() { - return getJobID(); - } // No Need - - @Override - public float getJobProgress() { - return 0; - } - - @Override - public Integer getErrCode() { - return null; - } - - @Override - public String getErrDesc() { - return null; - } - - @Override - public boolean isSuccess() { - return success; - } - - @Override - public void setSuccess(boolean success) { - this.success = success; - } - - @Override - public void updateByOperResult(LinkisOperResultAdapter adapter) { - // No need - } - - @Override - public LinkisLogData clone() throws CloneNotSupportedException { - throw new CloneNotSupportedException(); - // return null; - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/operator/OperatorUtils.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/operator/OperatorUtils.java deleted file mode 100644 index 767d44aea1..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/operator/OperatorUtils.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.application.operator; - -public class OperatorUtils { - public static int getNumOfLines(String str) { - if (str == null || str.length() == 0) { - return 0; - } - int lines = 1; - int len = str.length(); - for (int pos = 0; pos < len; pos++) { - char c = str.charAt(pos); - if (c == '\r') { - lines++; - if (pos + 1 < len && str.charAt(pos + 1) == '\n') { - pos++; - } - } else if (c == '\n') { - lines++; - } - } - return lines; - } - - public static int getFirstIndexSkippingLines(String str, Integer lines) { - if (str == null || str.length() == 0 || lines < 0) { - return -1; - } - if (lines == 0) { - return 0; - } - - int curLineIdx = 0; - int len = str.length(); - for (int pos = 0; pos < len; pos++) { - char c = str.charAt(pos); - if (c == '\r') { - curLineIdx++; - if (pos + 1 < len && str.charAt(pos + 1) == '\n') { - pos++; - } - } else if (c == '\n') { - curLineIdx++; - } else { - continue; - } - - if (curLineIdx >= lines) { - return pos + 1; - } - } - return -1; - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/present/LinkisResultInfoPresenter.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/present/LinkisResultInfoPresenter.java deleted file mode 100644 index 5a953fa5d7..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/present/LinkisResultInfoPresenter.java +++ /dev/null @@ -1,114 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.application.present; - -import org.apache.linkis.cli.application.present.model.LinkisResultInfoModel; -import org.apache.linkis.cli.common.entity.present.Model; -import org.apache.linkis.cli.common.entity.present.PresentWay; -import org.apache.linkis.cli.common.entity.present.Presenter; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.exception.PresenterException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; -import org.apache.linkis.cli.core.present.PresentModeImpl; -import org.apache.linkis.cli.core.present.display.DisplayOperFactory; -import org.apache.linkis.cli.core.present.display.data.StdoutDisplayData; -import org.apache.linkis.cli.core.utils.LogUtils; - -import org.apache.commons.lang3.StringUtils; - -public class LinkisResultInfoPresenter implements Presenter { - @Override - public void present(Model model, PresentWay presentWay) { - if (!(model instanceof LinkisResultInfoModel)) { - throw new PresenterException( - "PST0001", - ErrorLevel.ERROR, - CommonErrMsg.PresenterErr, - "Input model for \"LinkisResultInfoPresenter\" is not instance of \"LinkisResultInfoModel\""); - } - DisplayOperFactory.getDisplayOper(PresentModeImpl.STDOUT) - .doOutput(new StdoutDisplayData(formatResultIndicator((LinkisResultInfoModel) model))); - } - - protected String formatResultIndicator(LinkisResultInfoModel model) { - StringBuilder infoBuilder = new StringBuilder(); - String extraMsgStr = ""; - - if (model.getMessage() != null) { - extraMsgStr = model.getMessage().toString(); - } - if (model.getJobStatus().isJobSuccess()) { - - LogUtils.getInformationLogger().info("Job execute successfully! Will try get execute result"); - infoBuilder - .append("============Result:================") - .append(System.lineSeparator()) - .append("TaskId:") - .append(model.getJobID()) - .append(System.lineSeparator()) - .append("ExecId: ") - .append(model.getExecID()) - .append(System.lineSeparator()) - .append("User:") - .append(model.getUser()) - .append(System.lineSeparator()) - .append("Current job status:") - .append(model.getJobStatus()) - .append(System.lineSeparator()) - .append("extraMsg: ") - .append(extraMsgStr) - .append(System.lineSeparator()) - .append("result: ") - .append(extraMsgStr) - .append(System.lineSeparator()); - } else if (model.getJobStatus().isJobFinishedState()) { - LogUtils.getInformationLogger().info("Job failed! Will not try get execute result."); - infoBuilder - .append("============Result:================") - .append(System.lineSeparator()) - .append("TaskId:") - .append(model.getJobID()) - .append(System.lineSeparator()) - .append("ExecId: ") - .append(model.getExecID()) - .append(System.lineSeparator()) - .append("User:") - .append(model.getUser()) - .append(System.lineSeparator()) - .append("Current job status:") - .append(model.getJobStatus()) - .append(System.lineSeparator()) - .append("extraMsg: ") - .append(extraMsgStr) - .append(System.lineSeparator()); - if (model.getErrCode() != null) { - infoBuilder.append("errCode: ").append(model.getErrCode()).append(System.lineSeparator()); - } - if (StringUtils.isNotBlank(model.getErrDesc())) { - infoBuilder.append("errDesc: ").append(model.getErrDesc()).append(System.lineSeparator()); - } - } else { - throw new PresenterException( - "PST0011", - ErrorLevel.ERROR, - CommonErrMsg.PresenterErr, - "Job is not completed but triggered ResultPresenter"); - } - return infoBuilder.toString(); - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/present/LinkisResultPresenter.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/present/LinkisResultPresenter.java deleted file mode 100644 index b5e39c4e2e..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/present/LinkisResultPresenter.java +++ /dev/null @@ -1,232 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.application.present; - -import org.apache.linkis.cli.application.constants.AppConstants; -import org.apache.linkis.cli.application.interactor.job.data.LinkisResultSet; -import org.apache.linkis.cli.application.present.model.LinkisResultModel; -import org.apache.linkis.cli.common.entity.present.Model; -import org.apache.linkis.cli.common.entity.present.PresentWay; -import org.apache.linkis.cli.common.entity.present.Presenter; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.exception.PresenterException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; -import org.apache.linkis.cli.core.present.PresentModeImpl; -import org.apache.linkis.cli.core.present.PresentWayImpl; -import org.apache.linkis.cli.core.present.display.DisplayOperFactory; -import org.apache.linkis.cli.core.present.display.DisplayOperator; -import org.apache.linkis.cli.core.present.display.data.FileDisplayData; -import org.apache.linkis.cli.core.utils.CommonUtils; -import org.apache.linkis.cli.core.utils.LogUtils; - -import org.apache.commons.lang3.StringUtils; - -import java.text.MessageFormat; -import java.util.*; - -public class LinkisResultPresenter implements Presenter { - - @Override - public void present(Model model, PresentWay presentWay) { - if (!(model instanceof LinkisResultModel)) { - throw new PresenterException( - "PST0001", - ErrorLevel.ERROR, - CommonErrMsg.PresenterErr, - "Input model for \"LinkisResultPresenter\" is not instance of \"LinkisResultModel\""); - } - if (!(presentWay instanceof PresentWayImpl)) { - throw new PresenterException( - "PST0002", - ErrorLevel.ERROR, - CommonErrMsg.PresenterErr, - "Input PresentWay for \"LinkisResultPresenter\" is not instance of \"PresentWayImpl\""); - } - LinkisResultModel resultModel = (LinkisResultModel) model; - PresentWayImpl presentWay1 = (PresentWayImpl) presentWay; - - if (!resultModel.getJobStatus().isJobSuccess()) { - LogUtils.getInformationLogger() - .info("JobStatus is not \'success\'. Will not retrieve result-set."); - return; - } - String msg = ""; - if (resultModel.hasResult()) { - msg = - "Retrieving result-set, may take time if result-set is large, please do not exit program."; - } else { - msg = "Your job has no result."; - } - LogUtils.getInformationLogger().info(msg); - - final DisplayOperator displayOperator = - DisplayOperFactory.getDisplayOper( - presentWay1.getMode()); // currently we don't allow printing log to file here - - int preIdx = -1; - StringBuilder resultSb = new StringBuilder(); - - while (!resultModel.resultFinReceived()) { - preIdx = presentOneIteration(resultModel, preIdx, presentWay1, resultSb, displayOperator); - CommonUtils.doSleepQuietly(500l); - } - presentOneIteration(resultModel, preIdx, presentWay1, resultSb, displayOperator); - - if (presentWay1.getMode() == PresentModeImpl.TEXT_FILE - || StringUtils.isNotBlank(presentWay1.getPath())) { - LogUtils.getInformationLogger() - .info("ResultSet has been successfully written to path: " + presentWay1.getPath()); - } - } - - protected int presentOneIteration( - LinkisResultModel resultModel, - int preIdx, - PresentWayImpl presentWay, - StringBuilder resultSb, - DisplayOperator displayOperator) { - List linkisResultSets = resultModel.consumeResultContent(); - if (linkisResultSets != null && !linkisResultSets.isEmpty()) { - for (LinkisResultSet c : linkisResultSets) { - int idxResultset = c.getResultsetIdx(); - /** - * Notice: we assume result-sets are visited one by one in non-descending order!!! i.e. - * either idxResultset == preIdx or idxResultset - preIdx == 1 i.e. resultsets[0] -> - * resultsets[1] -> ... - */ - if (idxResultset - preIdx != 0 && idxResultset - preIdx != 1) { - throw new PresenterException( - "PST0002", - ErrorLevel.ERROR, - CommonErrMsg.PresenterErr, - "Linkis resultsets are visited in descending order or are not visited one-by-one"); - } - - boolean flag = idxResultset > preIdx; - if (presentWay.isDisplayMetaAndLogo()) { - if (idxResultset - preIdx == 1) { - resultSb.setLength(0); - resultSb - .append(MessageFormat.format(AppConstants.RESULTSET_LOGO, idxResultset + 1)) - .append(System.lineSeparator()); - if (c.getResultMeta() != null) { - resultSb - .append(AppConstants.RESULTSET_META_BEGIN_LOGO) - .append(System.lineSeparator()); - resultSb.append(formatResultMeta(c.getResultMeta())); - resultSb.append(AppConstants.RESULTSET_META_END_LOGO).append(System.lineSeparator()); - } - } - } - preIdx = idxResultset; - String contentStr = formatResultContent(c.getResultMeta(), c.getContent()); - if (contentStr != null) { - resultSb.append(contentStr); - } - if (resultSb.length() != 0) { - String resultFileName = - resultModel.getUser() - + "-task-" - + resultModel.getJobID() - + "-result-" - + String.valueOf(idxResultset + 1) - + ".txt"; - displayOperator.doOutput( - new FileDisplayData(presentWay.getPath(), resultFileName, resultSb.toString(), flag)); - resultSb.setLength(0); - } - } - } - return preIdx; - } - - protected String formatResultMeta(List> metaData) { - - StringBuilder outputBuilder = new StringBuilder(); - - if (metaData == null || metaData.size() == 0) { - return null; - } - - List titles = new ArrayList<>(); - - // gather keys as title - for (LinkedHashMap mapElement : metaData) { - if (mapElement == null || mapElement.size() == 0) { - continue; - } - - Set> entrySet = mapElement.entrySet(); - if (entrySet == null) { - break; - } - for (Map.Entry entry : entrySet) { - String key = entry.getKey(); - if (key != null && !titles.contains(key)) { - titles.add(key); - outputBuilder.append(key).append("\t"); - } - } - } - - outputBuilder.append(System.lineSeparator()); - - // gather value and print to output - for (LinkedHashMap mapElement : metaData) { - if (mapElement == null || mapElement.size() == 0) { - continue; - } - String candidate; - for (String title : titles) { - if (mapElement.containsKey(title)) { - candidate = mapElement.get(title); - } else { - candidate = "NULL"; - } - outputBuilder.append(candidate).append("\t"); - } - outputBuilder.append(System.lineSeparator()); - } - return outputBuilder.toString(); - } - - protected String formatResultContent( - List> metaData, List> contentData) { - - StringBuilder outputBuilder = new StringBuilder(); - if (contentData == null || contentData.size() == 0) { // finished - return null; - } - - int listLen = contentData.size(); - for (int i = 0; i < listLen; i++) { - List listElement = contentData.get(i); - if (listElement == null || listElement.size() == 0) { - continue; - } - for (String element : listElement) { - outputBuilder.append(element).append("\t"); - } - if (i < listLen - 1) { - outputBuilder.append(System.lineSeparator()); - } - } - - return outputBuilder.toString(); - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/present/model/LinkisJobInfoModel.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/present/model/LinkisJobInfoModel.java deleted file mode 100644 index acfc392e48..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/present/model/LinkisJobInfoModel.java +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.application.present.model; - -import org.apache.linkis.cli.application.interactor.job.data.LinkisJobDataImpl; -import org.apache.linkis.cli.common.entity.job.JobStatus; -import org.apache.linkis.cli.common.entity.present.Model; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.exception.TransformerException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; - -import org.apache.commons.lang3.exception.ExceptionUtils; - -import java.util.Date; - -public class LinkisJobInfoModel implements Model { - - private String cid; - private String jobId; - private String message; - private String exception; - private String cause; - - private String taskID; - private String instance; - private String simpleExecId; - private String execId; - private String umUser; - private String executionCode; - private String logPath; - private JobStatus status; - private String engineType; - private String runType; - private Long costTime; - private Date createdTime; - private Date updatedTime; - private Date engineStartTime; - private Integer errCode; - private String errMsg; - private String executeApplicationName; - private String requestApplicationName; - private Float progress; - - @Override - public void buildModel(Object data) { - if (!(data instanceof LinkisJobDataImpl)) { - throw new TransformerException( - "TFM0010", - ErrorLevel.ERROR, - CommonErrMsg.TransformerException, - "Failed to init LinkisJobInfoModel: " - + data.getClass().getCanonicalName() - + "is not instance of \"LinkisJobDataImpl\""); - } - this.jobId = ((LinkisJobDataImpl) data).getJobID(); - this.message = ((LinkisJobDataImpl) data).getMessage(); - this.taskID = ((LinkisJobDataImpl) data).getJobID(); - this.instance = ((LinkisJobDataImpl) data).getInstance(); - this.simpleExecId = ((LinkisJobDataImpl) data).getSimpleExecId(); - this.execId = ((LinkisJobDataImpl) data).getExecID(); - this.umUser = ((LinkisJobDataImpl) data).getUmUser(); - this.executionCode = ((LinkisJobDataImpl) data).getExecutionCode(); - this.logPath = ((LinkisJobDataImpl) data).getLogPath(); - this.status = ((LinkisJobDataImpl) data).getJobStatus(); - this.engineType = ((LinkisJobDataImpl) data).getEngineType(); - this.runType = ((LinkisJobDataImpl) data).getRunType(); - this.costTime = ((LinkisJobDataImpl) data).getCostTime(); - this.createdTime = ((LinkisJobDataImpl) data).getCreatedTime(); - this.updatedTime = ((LinkisJobDataImpl) data).getUpdatedTime(); - this.engineStartTime = ((LinkisJobDataImpl) data).getEngineStartTime(); - this.errCode = ((LinkisJobDataImpl) data).getErrCode(); - this.errMsg = ((LinkisJobDataImpl) data).getErrDesc(); - this.executeApplicationName = ((LinkisJobDataImpl) data).getExecuteApplicationName(); - this.requestApplicationName = ((LinkisJobDataImpl) data).getRequestApplicationName(); - this.progress = ((LinkisJobDataImpl) data).getJobProgress(); - Exception e = ((LinkisJobDataImpl) data).getException(); - if (e != null) { - this.exception = ExceptionUtils.getMessage(e); - this.cause = ExceptionUtils.getRootCauseMessage(e); - } - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/present/model/LinkisJobKillModel.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/present/model/LinkisJobKillModel.java deleted file mode 100644 index 95eb7d03c2..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/present/model/LinkisJobKillModel.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.application.present.model; - -import org.apache.linkis.cli.application.interactor.job.data.LinkisJobDataImpl; -import org.apache.linkis.cli.common.entity.job.JobStatus; -import org.apache.linkis.cli.common.entity.present.Model; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.exception.TransformerException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; - -import org.apache.commons.lang3.exception.ExceptionUtils; - -public class LinkisJobKillModel implements Model { - - private String cid; - private String jobId; - private String message; - private String exception; - private String cause; - - private String execID; - private String user; - private JobStatus jobStatus; - - @Override - public void buildModel(Object data) { - if (!(data instanceof LinkisJobDataImpl)) { - throw new TransformerException( - "TFM0010", - ErrorLevel.ERROR, - CommonErrMsg.TransformerException, - "Failed to init LinkisJobKillModel: " - + data.getClass().getCanonicalName() - + "is not instance of \"LinkisJobDataImpl\""); - } - this.jobId = ((LinkisJobDataImpl) data).getJobID(); - this.message = ((LinkisJobDataImpl) data).getMessage(); - this.execID = ((LinkisJobDataImpl) data).getExecID(); - this.user = ((LinkisJobDataImpl) data).getUser(); - this.jobStatus = ((LinkisJobDataImpl) data).getJobStatus(); - Exception e = ((LinkisJobDataImpl) data).getException(); - if (e != null) { - this.exception = ExceptionUtils.getMessage(e); - this.cause = ExceptionUtils.getRootCauseMessage(e); - } - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/present/model/LinkisResultInfoModel.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/present/model/LinkisResultInfoModel.java deleted file mode 100644 index bfa0ae6cc8..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/present/model/LinkisResultInfoModel.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.application.present.model; - -import org.apache.linkis.cli.application.interactor.job.data.LinkisResultData; -import org.apache.linkis.cli.common.entity.job.JobStatus; -import org.apache.linkis.cli.common.entity.present.Model; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.exception.TransformerException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; - -public class LinkisResultInfoModel implements Model { - private String jobID; - private String execID; - private String user; - private JobStatus jobStatus; - private String message; - private Integer errCode; - private String errDesc; - - @Override - public void buildModel(Object data) { - if (!(data instanceof LinkisResultData)) { - throw new TransformerException( - "TFM0010", - ErrorLevel.ERROR, - CommonErrMsg.TransformerException, - "Failed to init LinkisResultInfoModel: " - + data.getClass().getCanonicalName() - + "is not instance of \"LinkisResultData\""); - } - LinkisResultData jobData = (LinkisResultData) data; - jobID = jobData.getJobID(); - execID = jobData.getExecID(); - user = jobData.getUser(); - jobStatus = jobData.getJobStatus(); - message = jobData.getMessage(); - errCode = jobData.getErrCode(); - errDesc = jobData.getErrDesc(); - } - - public String getJobID() { - return jobID; - } - - public String getExecID() { - return execID; - } - - public String getUser() { - return user; - } - - public JobStatus getJobStatus() { - return jobStatus; - } - - public String getMessage() { - return message; - } - - public Integer getErrCode() { - return errCode; - } - - public String getErrDesc() { - return errDesc; - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/present/model/LinkisSubmitResultModel.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/present/model/LinkisSubmitResultModel.java deleted file mode 100644 index 6b622e1453..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/present/model/LinkisSubmitResultModel.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.application.present.model; - -import org.apache.linkis.cli.application.interactor.job.data.LinkisJobDataImpl; -import org.apache.linkis.cli.common.entity.job.JobStatus; -import org.apache.linkis.cli.common.entity.present.Model; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.exception.TransformerException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; - -import org.apache.commons.lang3.exception.ExceptionUtils; - -public class LinkisSubmitResultModel implements Model { - private String jobId; - private JobStatus status; - private String message; - private String exception; - private String cause; - - @Override - public void buildModel(Object data) { - if (!(data instanceof LinkisJobDataImpl)) { - throw new TransformerException( - "TFM0010", - ErrorLevel.ERROR, - CommonErrMsg.TransformerException, - "Failed to init LinkisJobInfoModel: " - + data.getClass().getCanonicalName() - + "is not instance of \"LinkisJobDataImpl\""); - } - this.jobId = ((LinkisJobDataImpl) data).getJobID(); - this.status = ((LinkisJobDataImpl) data).getJobStatus(); - this.message = ((LinkisJobDataImpl) data).getMessage(); - Exception e = ((LinkisJobDataImpl) data).getException(); - if (e != null) { - this.exception = ExceptionUtils.getMessage(e); - this.cause = ExceptionUtils.getRootCauseMessage(e); - } - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/suite/ExecutionSuite.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/suite/ExecutionSuite.java deleted file mode 100644 index 8b05a7cf78..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/suite/ExecutionSuite.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.application.suite; - -import org.apache.linkis.cli.common.entity.execution.Execution; -import org.apache.linkis.cli.common.entity.job.Job; -import org.apache.linkis.cli.common.entity.result.ResultHandler; - -import java.util.Map; - -public class ExecutionSuite { - Execution execution; - Map jobs; - ResultHandler[] resultHandlers; - - public ExecutionSuite( - Execution execution, Map jobs, ResultHandler... resultHandlers) { - this.execution = execution; - this.jobs = jobs; - this.resultHandlers = resultHandlers; - } - - public Execution getExecution() { - return execution; - } - - public void setExecution(Execution execution) { - this.execution = execution; - } - - public Map getJobs() { - return jobs; - } - - public void setJobs(Map jobs) { - this.jobs = jobs; - } - - public ResultHandler[] getResultHandlers() { - return resultHandlers; - } - - public void setResultHandlers(ResultHandler[] resultHandlers) { - this.resultHandlers = resultHandlers; - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/suite/ExecutionSuiteFactory.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/suite/ExecutionSuiteFactory.java deleted file mode 100644 index c55ee7b9e4..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/suite/ExecutionSuiteFactory.java +++ /dev/null @@ -1,225 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.application.suite; - -import org.apache.linkis.cli.application.constants.AppConstants; -import org.apache.linkis.cli.application.constants.AppKeys; -import org.apache.linkis.cli.application.interactor.command.LinkisCmdType; -import org.apache.linkis.cli.application.interactor.job.builder.LinkisManageJobBuilder; -import org.apache.linkis.cli.application.interactor.job.builder.LinkisOnceJobBuilder; -import org.apache.linkis.cli.application.interactor.job.builder.LinkisSubmitJobBuilder; -import org.apache.linkis.cli.application.interactor.job.subtype.LinkisManSubType; -import org.apache.linkis.cli.application.interactor.job.subtype.LinkisSubmitSubType; -import org.apache.linkis.cli.application.interactor.validate.LinkisManageValidator; -import org.apache.linkis.cli.application.interactor.validate.LinkisOnceSubmitValidator; -import org.apache.linkis.cli.application.interactor.validate.LinkisSubmitValidator; -import org.apache.linkis.cli.application.present.LinkisLogPresenter; -import org.apache.linkis.cli.application.present.LinkisResultInfoPresenter; -import org.apache.linkis.cli.application.present.LinkisResultPresenter; -import org.apache.linkis.cli.application.present.model.LinkisJobInfoModel; -import org.apache.linkis.cli.application.present.model.LinkisJobKillModel; -import org.apache.linkis.cli.application.present.model.LinkisResultInfoModel; -import org.apache.linkis.cli.application.present.model.LinkisResultModel; -import org.apache.linkis.cli.application.utils.Utils; -import org.apache.linkis.cli.common.entity.command.CmdType; -import org.apache.linkis.cli.common.entity.execution.Execution; -import org.apache.linkis.cli.common.entity.job.Job; -import org.apache.linkis.cli.common.entity.job.JobData; -import org.apache.linkis.cli.common.entity.job.JobDescription; -import org.apache.linkis.cli.common.entity.job.JobSubType; -import org.apache.linkis.cli.common.entity.operator.JobOperator; -import org.apache.linkis.cli.common.entity.present.PresentWay; -import org.apache.linkis.cli.common.entity.result.ResultHandler; -import org.apache.linkis.cli.common.entity.validate.Validator; -import org.apache.linkis.cli.common.entity.var.VarAccess; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.exception.LinkisClientExecutionException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; -import org.apache.linkis.cli.core.interactor.execution.AsyncSubmission; -import org.apache.linkis.cli.core.interactor.execution.Help; -import org.apache.linkis.cli.core.interactor.execution.JobManagement; -import org.apache.linkis.cli.core.interactor.execution.SyncSubmission; -import org.apache.linkis.cli.core.interactor.job.JobBuilder; -import org.apache.linkis.cli.core.interactor.result.DefaultResultHandler; -import org.apache.linkis.cli.core.interactor.result.PresentResultHandler; -import org.apache.linkis.cli.core.present.DefaultStdOutPresenter; - -import org.apache.commons.lang3.StringUtils; - -import java.util.HashMap; -import java.util.Map; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class ExecutionSuiteFactory { - private static Logger logger = LoggerFactory.getLogger(ExecutionSuiteFactory.class); - - public static ExecutionSuite getSuite( - CmdType cmdType, VarAccess stdVarAccess, VarAccess sysVarAccess) { - - ExecutionSuite suite; - Execution execution; - Map jobs = new HashMap<>(); - JobSubType subType; - JobBuilder jobBuilder; - Validator validator; - ResultHandler defaultHandler = new DefaultResultHandler(); - - /* - Prepare Builders and command-specific components - */ - if (cmdType == LinkisCmdType.UNIVERSAL) { - if (stdVarAccess.hasVar(AppKeys.LINKIS_CLIENT_KILL_OPT)) { - execution = new JobManagement(); - jobBuilder = new LinkisManageJobBuilder(); - PresentResultHandler handler = new PresentResultHandler(); - handler.setPresenter(new DefaultStdOutPresenter()); - handler.setModel(new LinkisJobKillModel()); - validator = new LinkisManageValidator(); - subType = LinkisManSubType.KILL; - suite = new ExecutionSuite(execution, jobs, handler, defaultHandler); - } else if (stdVarAccess.hasVar(AppKeys.LINKIS_CLIENT_STATUS_OPT)) { - execution = new JobManagement(); - jobBuilder = new LinkisManageJobBuilder(); - PresentResultHandler handler = new PresentResultHandler(); - handler.setPresenter(new DefaultStdOutPresenter()); - handler.setModel(new LinkisJobInfoModel()); - validator = new LinkisManageValidator(); - subType = LinkisManSubType.STATUS; - suite = new ExecutionSuite(execution, jobs, handler, defaultHandler); - } else if (stdVarAccess.hasVar(AppKeys.LINKIS_CLIENT_LOG_OPT)) { - execution = new JobManagement(); - jobBuilder = new LinkisManageJobBuilder().setLogListener(new LinkisLogPresenter()); - validator = new LinkisManageValidator(); - subType = LinkisManSubType.LOG; - suite = new ExecutionSuite(execution, jobs, null, defaultHandler); - } else if (stdVarAccess.hasVar(AppKeys.LINKIS_CLIENT_RESULT_OPT)) { - execution = new JobManagement(); - jobBuilder = new LinkisManageJobBuilder(); - PresentResultHandler handler = new PresentResultHandler(); - handler.setPresenter(new LinkisResultPresenter()); - handler.setModel(new LinkisResultModel()); - validator = new LinkisManageValidator(); - subType = LinkisManSubType.RESULT; - suite = new ExecutionSuite(execution, jobs, handler, defaultHandler); - } else if (stdVarAccess.hasVar(AppKeys.LINKIS_CLIENT_HELP_OPT)) { - execution = new Help(); - jobs.put( - "help", - new Job() { - @Override - public String getCid() { - return null; - } - - @Override - public CmdType getCmdType() { - return cmdType; - } - - @Override - public JobSubType getSubType() { - return null; - } - - @Override - public JobDescription getJobDesc() { - return null; - } - - @Override - public JobData getJobData() { - return null; - } - - @Override - public JobOperator getJobOperator() { - return null; - } - - @Override - public PresentWay getPresentWay() { - return null; - } - }); - return new ExecutionSuite(execution, jobs, null, defaultHandler); - } else { - Boolean asyncSubmission = - stdVarAccess.getVarOrDefault(Boolean.class, AppKeys.LINKIS_CLIENT_ASYNC_OPT, false); - if (asyncSubmission) { - execution = new AsyncSubmission(); - PresentResultHandler handler = new PresentResultHandler(); - handler.setPresenter(new DefaultStdOutPresenter()); - handler.setModel(new LinkisJobInfoModel()); - jobBuilder = new LinkisSubmitJobBuilder().setAsync(true); - subType = LinkisSubmitSubType.SUBMIT; - suite = new ExecutionSuite(execution, jobs, handler, defaultHandler); - validator = new LinkisSubmitValidator(); - } else { - execution = new SyncSubmission(); - subType = LinkisSubmitSubType.SUBMIT; - PresentResultHandler handler1 = new PresentResultHandler(); - handler1.setPresenter(new LinkisResultInfoPresenter()); - handler1.setModel(new LinkisResultInfoModel()); - PresentResultHandler handler2 = new PresentResultHandler(); - handler2.setPresenter(new LinkisResultPresenter()); - handler2.setModel(new LinkisResultModel()); - - String mode = - stdVarAccess.getVarOrDefault( - String.class, AppKeys.LINKIS_CLIENT_MODE_OPT, AppConstants.UJES_MODE); - if (StringUtils.equalsIgnoreCase(mode, AppConstants.ONCE_MODE)) { - jobBuilder = new LinkisOnceJobBuilder().setLogListener(new LinkisLogPresenter()); - ; - validator = new LinkisOnceSubmitValidator(); - } else { - jobBuilder = new LinkisSubmitJobBuilder().setLogListener(new LinkisLogPresenter()); - validator = new LinkisSubmitValidator(); - } - suite = new ExecutionSuite(execution, jobs, handler1, handler2, defaultHandler); - } - } - /* - build job - */ - Job job = - jobBuilder - .setCid(AppConstants.DUMMY_CID) // currently we don't need this - .setCmdType(cmdType) - .setJobSubType(subType) - .setStdVarAccess(stdVarAccess) - .setSysVarAccess(sysVarAccess) - .build(); - logger.info("==========JOB============\n" + Utils.GSON.toJson(job.getJobDesc())); - if (validator != null) { - validator.doValidation(job); - } - - jobs.put(job.getCid(), job); - - return suite; - } else { - throw new LinkisClientExecutionException( - "EXE0029", - ErrorLevel.ERROR, - CommonErrMsg.ExecutionInitErr, - "Command Type is not supported"); - } - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/utils/Utils.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/utils/Utils.java deleted file mode 100644 index e70c32a37b..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/utils/Utils.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.application.utils; - -import org.apache.commons.lang3.StringUtils; - -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class Utils { - - public static final Gson GSON = - new GsonBuilder().setPrettyPrinting().disableHtmlEscaping().create(); - private static final Logger logger = LoggerFactory.getLogger(Utils.class); - - public static boolean isValidExecId(String execId) { - boolean ret = false; - if (StringUtils.isNotBlank(execId)) { - ret = true; - } - return ret; - } - - public static String progressInPercentage(float progress) { - return String.valueOf(progress * 100) + "%"; - } - - public static void doSleepQuietly(Long sleepMills) { - try { - Thread.sleep(sleepMills); - } catch (Exception ignore) { - // ignored - } - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/interactor/command/template/ProcessInputUtil.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/interactor/command/template/ProcessInputUtil.java deleted file mode 100644 index 33af32bb4b..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/interactor/command/template/ProcessInputUtil.java +++ /dev/null @@ -1,127 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.application.interactor.command.template; - -import org.apache.linkis.cli.application.constants.AppConstants; -import org.apache.linkis.cli.application.constants.AppKeys; -import org.apache.linkis.cli.application.data.ProcessedData; -import org.apache.linkis.cli.application.utils.Utils; -import org.apache.linkis.cli.common.entity.command.CmdTemplate; -import org.apache.linkis.cli.common.entity.command.Params; -import org.apache.linkis.cli.common.entity.properties.ClientProperties; -import org.apache.linkis.cli.common.entity.var.VarAccess; -import org.apache.linkis.cli.core.constants.CommonConstants; -import org.apache.linkis.cli.core.interactor.command.fitter.SingleTplFitter; -import org.apache.linkis.cli.core.interactor.command.parser.Parser; -import org.apache.linkis.cli.core.interactor.command.parser.SingleCmdParser; -import org.apache.linkis.cli.core.interactor.command.parser.result.ParseResult; -import org.apache.linkis.cli.core.interactor.properties.PropertiesLoader; -import org.apache.linkis.cli.core.interactor.properties.PropsFilesScanner; -import org.apache.linkis.cli.core.interactor.properties.StdPropsLoader; -import org.apache.linkis.cli.core.interactor.properties.reader.PropertiesReader; -import org.apache.linkis.cli.core.interactor.properties.reader.PropsFileReader; -import org.apache.linkis.cli.core.interactor.properties.reader.SysEnvReader; -import org.apache.linkis.cli.core.interactor.properties.reader.SysPropsReader; -import org.apache.linkis.cli.core.interactor.validate.ParsedTplValidator; -import org.apache.linkis.cli.core.interactor.var.StdVarAccess; -import org.apache.linkis.cli.core.interactor.var.SysVarAccess; -import org.apache.linkis.cli.core.utils.LogUtils; - -import org.apache.commons.lang3.StringUtils; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class ProcessInputUtil { - private static Logger logger = LoggerFactory.getLogger(ProcessInputUtil.class); - - public static ProcessedData generateProcessedData(String[] cmdStr, CmdTemplate template) { - /* - user input - */ - Parser parser = - new SingleCmdParser() - .setMapper(null) - .setTemplate(template) - .setFitter(new SingleTplFitter()); - - ParseResult result = parser.parse(cmdStr); - - ParsedTplValidator parsedTplValidator = new ParsedTplValidator(); - parsedTplValidator.doValidation(result.getParsedTemplate()); - - Params params = result.getParams(); - logger.debug("==========params============\n" + Utils.GSON.toJson(params)); - - Map propertiesMap = new HashMap<>(); - /* - default config, -Dconf.root & -Dconf.file specifies config path - */ - System.setProperty("conf.root", "src/test/resources/conf/"); - System.setProperty("conf.file", "linkis-cli.properties"); - String configPath = System.getProperty("conf.root"); - String defaultConfFileName = System.getProperty("conf.file"); - List readersList = - new PropsFilesScanner().getPropsReaders(configPath); // +1 user config - /* - user defined config - */ - String userConfPath = null; - if (params.containsParam(AppKeys.LINKIS_CLIENT_USER_CONFIG)) { - userConfPath = - (String) params.getParamItemMap().get(AppKeys.LINKIS_CLIENT_USER_CONFIG).getValue(); - } - if (StringUtils.isNotBlank(userConfPath)) { - PropertiesReader reader = - new PropsFileReader() - .setPropsId(AppKeys.LINKIS_CLIENT_USER_CONFIG) - .setPropsPath(userConfPath); - readersList.add(reader); - } else { - LogUtils.getInformationLogger() - .info("User does not provide usr-configuration file. Will use default config"); - } - readersList.add(new SysPropsReader()); - readersList.add(new SysEnvReader()); - PropertiesLoader loader = - new StdPropsLoader() - .addPropertiesReaders(readersList.toArray(new PropertiesReader[readersList.size()])); - ClientProperties[] loaderResult = loader.loadProperties(); - for (ClientProperties properties : loaderResult) { - propertiesMap.put(properties.getPropsId(), properties); - } - - VarAccess stdVarAccess = - new StdVarAccess() - .setCmdParams(params) - .setUserConf(propertiesMap.get(AppKeys.LINKIS_CLIENT_USER_CONFIG)) - .setDefaultConf(propertiesMap.get(AppConstants.DEFAULT_CONFIG_NAME)) - .init(); - - VarAccess sysVarAccess = - new SysVarAccess() - .setSysProp(propertiesMap.get(CommonConstants.SYSTEM_PROPERTIES_IDENTIFIER)) - .setSysEnv(propertiesMap.get(CommonConstants.SYSTEM_ENV_IDENTIFIER)); - - return new ProcessedData(null, params.getCmdType(), stdVarAccess, sysVarAccess); - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/interactor/execution/LinkisExecutorTest.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/interactor/execution/LinkisExecutorTest.java deleted file mode 100644 index b2571049d3..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/interactor/execution/LinkisExecutorTest.java +++ /dev/null @@ -1,113 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.application.interactor.execution; - -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -public class LinkisExecutorTest { - - @BeforeEach - public void before() {} - - @AfterEach - public void after() {} - - /** Method: setDriver(LinkisClientDriver driver) */ - @Test - public void testSetDriver() { - // TODO: Test goes here... - } - - /** Method: setDriverTransformer(DriverTransformer driverTransformer) */ - @Test - public void testSetDriverTransformer() { - // TODO: Test goes here... - } - - /** Method: getJobID(JobSubmitExec execData) */ - @Test - public void testGetJobID() { - // TODO: Test goes here... - } - - /** Method: submit(Job job) */ - @Test - public void testDoAsyncSubmit() { - // TODO: Test goes here... - } - - /** Method: updateJobStatus(JobSubmitExec execData) */ - @Test - public void testDoUpdateProgress() { - // TODO: Test goes here... - } - - /** Method: checkSubmit(JobSubmitExec execData) */ - @Test - public void testDoCheckSubmit() { - // TODO: Test goes here... - } - - /** Method: doGetFinalResult(JobSubmitExec execData) */ - @Test - public void testDoGetFinalResult() { - // TODO: Test goes here... - } - - /** Method: checkInit() */ - @Test - public void testCheckInit() { - // TODO: Test goes here... - } - - /** Method: doKill(Job job) */ - @Test - public void testDoKill() { - // TODO: Test goes here... - } - - /** Method: doQuery(Job job) */ - @Test - public void testDoQuery() { - // TODO: Test goes here... - } - - /** Method: doTransform(Job job) */ - @Test - public void testDoTransform() { - // TODO: Test goes here... - } - - /** Method: updateExecDataByDwsResult(JobSubmitExec execData, DWSResult result) */ - @Test - public void testUpdateExecDataByDwsResult() { - // TODO: Test goes here... - /* - try { - Method method = LinkisSubmitExecutor.getClass().getMethod("updateExecDataByDwsResult", JobSubmitExec.class, DWSResult.class); - method.setAccessible(true); - method.invoke(, ); - } catch(NoSuchMethodException e) { - } catch(IllegalAccessException e) { - } catch(InvocationTargetException e) { - } - */ - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/interactor/job/LinkisJobBuilderTest.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/interactor/job/LinkisJobBuilderTest.java deleted file mode 100644 index 99836e8ec6..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/interactor/job/LinkisJobBuilderTest.java +++ /dev/null @@ -1,106 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.application.interactor.job; - -import org.apache.linkis.cli.application.data.ProcessedData; -import org.apache.linkis.cli.application.interactor.command.template.ProcessInputUtil; -import org.apache.linkis.cli.application.interactor.command.template.TestSparkCmdTemplate; -import org.apache.linkis.cli.core.interactor.job.JobBuilder; - -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -public class LinkisJobBuilderTest { - JobBuilder builder; - - @BeforeEach - public void before() { - String[] cmdStr = - new String[] { - "-u", - "hadoop", - "-pwd", - "1234", - "-c", - "src/test/resources/conf/user.properties", - "--cmd", - "show tables", - "--split", - "\',\'", - "--queue", - "q05", - "--name", - "testApp", - // "--hiveconf", "/path/...", - // "--num-executors", "4", - "--executor-cores", - "4", - "--executor-memory", - "4G", - "--shuffle-partitions", - "200", - "-confMap", - "kk=vv,kkk=vvv,spark.executor.instances=8" - }; - ProcessedData data = ProcessInputUtil.generateProcessedData(cmdStr, new TestSparkCmdTemplate()); - // builder = new LinkisJobBuilder() - // .setIdentifier(data.getIdentifier()) - // .setCmdType(data.getCmdType()) - // .setExecutionType(data.getExecutionType()) - // .setSubExecutionType(data.getSubExecutionType()) - // .setStdVarAccess(data.getStdVarAccess()) - // .setSysVarAccess(data.getSysVarAccess()); - } - - @AfterEach - public void after() {} - - /** Method: setIdentifier(String identifier) */ - @Test - public void testSetIdentifier() { - // TODO: Test goes here... - } - - /** Method: build() */ - @Test - public void testBuild() { - // TODO: Test goes here... - // LinkisJob job = builder.build(); - // System.out.println(CommonUtils.GSON.toJson(job)); - // assertEquals(job.getParamConfMap().get("spark.executor.instances"), "8"); - } - - /** Method: setStdVarAccess(VarAccess varAccess) */ - @Test - public void testSetStdVarAccess() { - // TODO: Test goes here... - } - - /** Method: setSysVarAccess(VarAccess varAccess) */ - @Test - public void testSetSysVarAccess() { - // TODO: Test goes here... - } - - /** Method: getTargetNewInstance() */ - @Test - public void testGetTargetNewInstance() { - // TODO: Test goes here... - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/job/Job.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/job/Job.java deleted file mode 100644 index 5339db3c09..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/job/Job.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.common.entity.job; - -import org.apache.linkis.cli.common.entity.command.CmdTemplate; -import org.apache.linkis.cli.common.entity.command.CmdType; -import org.apache.linkis.cli.common.entity.operator.JobOperator; -import org.apache.linkis.cli.common.entity.present.PresentWay; - -public interface Job { - /** Linkis-cli specified id, not server-side returned job-id/task-id */ - String getCid(); - - /** - * Command Type for this Job, should be able to use this to find out corresponding {@link - * CmdTemplate} - */ - CmdType getCmdType(); - - /** specifies which kind of sub-execution: e.g. jobManagement: status/list/log/kill; */ - JobSubType getSubType(); - - /** - * input-param/config will be stored in JobDescription information contained by this - * data-structure should be passed to server - */ - JobDescription getJobDesc(); - - /** - * data generated during execution(e.g. job status, job id, log, result etc.) is stored here - * information contained by this data-structure can be further passed to server - */ - JobData getJobData(); - - /** operates lower level components(usually encapsulates a client) */ - JobOperator getJobOperator(); - - /** decide how result should be presented */ - PresentWay getPresentWay(); -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/result/ResultHandler.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/result/ResultHandler.java deleted file mode 100644 index cf10cb2bf4..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/result/ResultHandler.java +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.common.entity.result; - -public interface ResultHandler { - void process(ExecutionResult executionResult); -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/exception/handler/ExceptionHandler.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/exception/handler/ExceptionHandler.java deleted file mode 100644 index cb5a756ac3..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/exception/handler/ExceptionHandler.java +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.common.exception.handler; - -public interface ExceptionHandler { - void handle(Exception e); -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/test/java/org/apache/linkis/cli/common/exception/error/ErrorLevelTest.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/test/java/org/apache/linkis/cli/common/exception/error/ErrorLevelTest.java deleted file mode 100644 index e6d14f71ea..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/test/java/org/apache/linkis/cli/common/exception/error/ErrorLevelTest.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.common.exception.error; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.Test; - -public class ErrorLevelTest { - - @Test - @DisplayName("enumTest") - public void enumTest() { - - int infoLevel = ErrorLevel.INFO.getLevel(); - int warnLevel = ErrorLevel.WARN.getLevel(); - int errorLevel = ErrorLevel.ERROR.getLevel(); - int fatalLevel = ErrorLevel.FATAL.getLevel(); - int retryLevel = ErrorLevel.RETRY.getLevel(); - - Assertions.assertTrue(0 == infoLevel); - Assertions.assertTrue(1 == warnLevel); - Assertions.assertTrue(2 == errorLevel); - Assertions.assertTrue(3 == fatalLevel); - Assertions.assertTrue(4 == retryLevel); - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/pom.xml b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/pom.xml deleted file mode 100644 index 4d453144fb..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/pom.xml +++ /dev/null @@ -1,43 +0,0 @@ - - - - 4.0.0 - - - org.apache.linkis - linkis-cli - ${revision} - - linkis-cli-core - jar - - - - org.apache.linkis - linkis-cli-common - ${project.version} - - - - org.reflections - reflections - - - - - diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/builder/AbstractBuilder.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/builder/AbstractBuilder.java deleted file mode 100644 index 8063662cb5..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/builder/AbstractBuilder.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.builder; - -public abstract class AbstractBuilder implements Builder { - protected T targetObj; - - public AbstractBuilder() { - reset(); - } - - protected void reset() { - targetObj = getTargetNewInstance(); - } - - @Override - public T build() { - T ret = targetObj; - reset(); - return ret; - } - - protected abstract T getTargetNewInstance(); - - // protected T getTargetNewInstance() { - // try { - // ParameterizedType pt = (ParameterizedType) this.getClass().getGenericSuperclass(); - // Class clazz = (Class) pt.getActualTypeArguments()[0]; - // targetObj = clazz.newInstance(); - // } catch (InstantiationException e) { - // throw new BuilderException("BLD0001", ErrorLevel.ERROR, - // CommonErrMsg.BuilderBuildErr, "Cannot generate new instance.", e); - // } catch (IllegalAccessException ie) { - // throw new BuilderException("BLD0001", ErrorLevel.ERROR, - // CommonErrMsg.BuilderBuildErr, "Cannot generate new instance.", ie); - // } - // return targetObj; - // } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/builder/BuildableByVarAccess.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/builder/BuildableByVarAccess.java deleted file mode 100644 index ca6454e923..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/builder/BuildableByVarAccess.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.builder; - -import org.apache.linkis.cli.common.entity.var.VarAccess; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.exception.BuilderException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; - -import java.lang.reflect.ParameterizedType; - -public abstract class BuildableByVarAccess extends AbstractBuilder { - protected VarAccess stdVarAccess; - protected VarAccess sysVarAccess; - - protected void checkInit() { - if (stdVarAccess == null || sysVarAccess == null) { - ParameterizedType pt = (ParameterizedType) this.getClass().getGenericSuperclass(); - Class clazz = (Class) pt.getActualTypeArguments()[0]; - throw new BuilderException( - "BLD0003", - ErrorLevel.ERROR, - CommonErrMsg.BuilderInitErr, - "Cannot init builder: " - + clazz.getCanonicalName() - + "Cause: stdVarAccess or sysVarAccess is null"); - } - stdVarAccess.checkInit(); - sysVarAccess.checkInit(); - } - - public BuildableByVarAccess setStdVarAccess(VarAccess varAccess) { - this.stdVarAccess = varAccess; - return this; - } - - public BuildableByVarAccess setSysVarAccess(VarAccess varAccess) { - this.sysVarAccess = varAccess; - return this; - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/builder/Builder.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/builder/Builder.java deleted file mode 100644 index e66ba2ee55..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/builder/Builder.java +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.builder; - -public interface Builder { - T build(); -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/constants/CommonConstants.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/constants/CommonConstants.java deleted file mode 100644 index 1362539158..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/constants/CommonConstants.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.constants; - -public class CommonConstants { - - public static final Long JOB_QUERY_SLEEP_MILLS = 2000l; - public static final Integer REQUEST_MAX_RETRY_TIME = 3; - - public static final String UNIVERSAL_SUBCMD = "linkis-cli"; - public static final String UNIVERSAL_SUBCMD_DESC = - "command for all types of jobs supported by Linkis"; - - public static final String SUCCESS_INDICATOR = "############Execute Success!!!########"; - public static final String FAILURE_INDICATOR = "############Execute Error!!!########"; - public static final String ARRAY_SEQ = "@#@"; - public static final String ARRAY_SEQ_REGEX = "(?=([^\"]*\"[^\"]*\")*[^\"]*$)"; - public static final int MAX_NUM_OF_COMMAND_ARGUEMENTS = 10; - public static final String CONFIG_DIR = "config.path"; - public static final String[] CONFIG_EXTENSION = {"properties"}; - public static final String SYSTEM_PROPERTIES_IDENTIFIER = "SYS_PROP"; - public static final String SYSTEM_ENV_IDENTIFIER = "SYS_ENV"; -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/constants/CommonKeys.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/constants/CommonKeys.java deleted file mode 100644 index 39eb2f9418..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/constants/CommonKeys.java +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.constants; - -public class CommonKeys {} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/exception/handler/CommandExceptionHandler.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/exception/handler/CommandExceptionHandler.java deleted file mode 100644 index 25d7a04d20..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/exception/handler/CommandExceptionHandler.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.exception.handler; - -import org.apache.linkis.cli.common.entity.command.CmdTemplate; -import org.apache.linkis.cli.common.exception.handler.ExceptionHandler; -import org.apache.linkis.cli.core.exception.CommandException; -import org.apache.linkis.cli.core.interactor.command.CmdTemplateFactory; -import org.apache.linkis.cli.core.present.HelpInfoPresenter; -import org.apache.linkis.cli.core.present.model.HelpInfoModel; - -public class CommandExceptionHandler implements ExceptionHandler { - @Override - public void handle(Exception e) { - if (e instanceof CommandException) { - if (((CommandException) e).requireHelp()) { - - CmdTemplate template = - CmdTemplateFactory.getTemplateOri(((CommandException) e).getCmdType()); - - if (template != null) { - HelpInfoModel model = new HelpInfoModel(); - model.buildModel(template); - - new HelpInfoPresenter().present(model, null); - } - } - } - new DefaultExceptionHandler().handle(e); - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/exception/handler/DefaultExceptionHandler.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/exception/handler/DefaultExceptionHandler.java deleted file mode 100644 index c83e711513..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/exception/handler/DefaultExceptionHandler.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.exception.handler; - -import org.apache.linkis.cli.common.exception.LinkisClientRuntimeException; -import org.apache.linkis.cli.common.exception.handler.ExceptionHandler; -import org.apache.linkis.cli.core.utils.LogUtils; - -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.exception.ExceptionUtils; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class DefaultExceptionHandler implements ExceptionHandler { - private static Logger logger = LoggerFactory.getLogger(DefaultExceptionHandler.class); - - @Override - public void handle(Exception exception) { - if (exception instanceof LinkisClientRuntimeException) { - LinkisClientRuntimeException e = (LinkisClientRuntimeException) exception; - switch (e.getLevel()) { - case INFO: - logger.info(e.getMessage(), e); - LogUtils.getInformationLogger().info(e.getMessage()); - break; - case WARN: - logger.warn(e.getMessage(), e); - LogUtils.getInformationLogger().warn(getErrInfoWithoutStacktrace(e)); - break; - case ERROR: - logger.error(e.getMessage(), e); - LogUtils.getInformationLogger().error(getErrInfoWithoutStacktrace(e)); - break; - case FATAL: - String msg = StringUtils.substringAfter(e.getMessage(), "[ERROR]"); - logger.error(msg, e); - LogUtils.getInformationLogger().error("[FATAL]" + msg, e); - System.exit(-1); - break; - } - - } else { - logger.error(exception.getMessage(), exception); - LogUtils.getInformationLogger().error(exception.getMessage(), exception); - } - } - - private String getErrInfoWithoutStacktrace(Exception e) { - if (e == null) { - return ""; - } - StringBuilder sb = new StringBuilder(); - if (e instanceof NullPointerException) { - sb.append(ExceptionUtils.getStackTrace(e)); - } else { - sb.append(e.getMessage()); - } - if (e.getCause() != null) { - sb.append(System.lineSeparator()) - .append("Caused by: ") - .append((e.getCause().getClass().getCanonicalName())) - .append(": ") - .append(e.getCause().getMessage()); - } - return sb.toString(); - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/execution/AsyncSubmission.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/execution/AsyncSubmission.java deleted file mode 100644 index 765c5969d4..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/execution/AsyncSubmission.java +++ /dev/null @@ -1,109 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.interactor.execution; - -import org.apache.linkis.cli.common.entity.execution.Execution; -import org.apache.linkis.cli.common.entity.job.Job; -import org.apache.linkis.cli.common.entity.result.ExecutionResult; -import org.apache.linkis.cli.common.entity.result.ExecutionStatus; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.constants.CommonConstants; -import org.apache.linkis.cli.core.exception.LinkisClientExecutionException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; -import org.apache.linkis.cli.core.interactor.job.AsyncBackendJob; -import org.apache.linkis.cli.core.interactor.result.ExecutionResultImpl; -import org.apache.linkis.cli.core.interactor.result.ExecutionStatusEnum; -import org.apache.linkis.cli.core.utils.CommonUtils; - -import java.util.Map; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Execute job asynchronously TODO: put exception during execution in ExecutionResult and do not - * interrupt execution - */ -public class AsyncSubmission implements Execution { - private static final Logger logger = LoggerFactory.getLogger(AsyncSubmission.class); - - @Override - public ExecutionResult execute(Map jobs) { - ExecutionStatus executionStatus; - Exception exception = null; // TODO - - if (jobs == null || jobs.size() == 0) { - throw new LinkisClientExecutionException( - "EXE0001", - ErrorLevel.ERROR, - CommonErrMsg.ExecutionInitErr, - "Null or empty Jobs is submitted to current execution"); - } - - if (jobs.size() > 1) { - throw new LinkisClientExecutionException( - "EXE0001", - ErrorLevel.ERROR, - CommonErrMsg.ExecutionInitErr, - "Multiple Jobs is not Supported by current execution"); - } - - Job job = jobs.get(jobs.keySet().toArray(new String[jobs.size()])[0]); - - if (!(job instanceof AsyncBackendJob)) { - throw new LinkisClientExecutionException( - "EXE0001", - ErrorLevel.ERROR, - CommonErrMsg.ExecutionInitErr, - "Backend for \"" + job.getClass().getCanonicalName() + "\" does not support async"); - } - - if (job.getSubType() == null) { - throw new LinkisClientExecutionException( - "EXE0001", - ErrorLevel.ERROR, - CommonErrMsg.ExecutionInitErr, - "SubExecType should not be null"); - } - - try { - ((AsyncBackendJob) job).submit(); - CommonUtils.doSleepQuietly(CommonConstants.JOB_QUERY_SLEEP_MILLS); - ((AsyncBackendJob) job).updateJobStatus(); - if (job.getJobData().getJobStatus().isJobSubmitted()) { - executionStatus = ExecutionStatusEnum.SUCCEED; - } else { - executionStatus = ExecutionStatusEnum.FAILED; - if (job.getJobData().getException() != null) { - exception = job.getJobData().getException(); - } - } - } catch (Exception e) { - exception = e; - executionStatus = ExecutionStatusEnum.FAILED; - logger.warn("Failed to submit job.", e); - } - - return new ExecutionResultImpl(jobs, executionStatus, exception); - } - - @Override - public boolean terminate(Map jobs) { - return true; - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/execution/Help.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/execution/Help.java deleted file mode 100644 index b2d34746c6..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/execution/Help.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.interactor.execution; - -import org.apache.linkis.cli.common.entity.execution.Execution; -import org.apache.linkis.cli.common.entity.job.Job; -import org.apache.linkis.cli.common.entity.result.ExecutionResult; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.exception.LinkisClientExecutionException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; -import org.apache.linkis.cli.core.interactor.command.CmdTemplateFactory; -import org.apache.linkis.cli.core.interactor.result.ExecutionResultImpl; -import org.apache.linkis.cli.core.interactor.result.ExecutionStatusEnum; -import org.apache.linkis.cli.core.present.HelpInfoPresenter; -import org.apache.linkis.cli.core.present.model.HelpInfoModel; - -import java.util.Map; - -public class Help implements Execution { - @Override - public ExecutionResult execute(Map jobs) { - - if (jobs.size() > 1) { - throw new LinkisClientExecutionException( - "EXE0001", - ErrorLevel.ERROR, - CommonErrMsg.ExecutionInitErr, - "Multiple Jobs is not Supported by current execution"); - } - - HelpInfoModel model = new HelpInfoModel(); - Job job = jobs.values().toArray(new Job[jobs.size()])[0]; - - model.buildModel(CmdTemplateFactory.getTemplateOri(job.getCmdType())); - - new HelpInfoPresenter().present(model, null); - return new ExecutionResultImpl(null, ExecutionStatusEnum.SUCCEED, null); - } - - @Override - public boolean terminate(Map jobs) { - return true; - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/execution/JobManagement.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/execution/JobManagement.java deleted file mode 100644 index c4741bdf68..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/execution/JobManagement.java +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.interactor.execution; - -import org.apache.linkis.cli.common.entity.execution.Execution; -import org.apache.linkis.cli.common.entity.job.Job; -import org.apache.linkis.cli.common.entity.result.ExecutionResult; -import org.apache.linkis.cli.common.entity.result.ExecutionStatus; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.exception.LinkisClientExecutionException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; -import org.apache.linkis.cli.core.interactor.job.ManagableBackendJob; -import org.apache.linkis.cli.core.interactor.result.ExecutionResultImpl; -import org.apache.linkis.cli.core.interactor.result.ExecutionStatusEnum; - -import java.util.Map; - -public class JobManagement implements Execution { - @Override - public ExecutionResult execute(Map jobs) { - ExecutionStatus executionStatus; - Exception exception = null; // TODO - - if (jobs == null || jobs.size() == 0) { - throw new LinkisClientExecutionException( - "EXE0001", - ErrorLevel.ERROR, - CommonErrMsg.ExecutionInitErr, - "Null or empty Jobs is submitted to current execution"); - } - - if (jobs.size() > 1) { - throw new LinkisClientExecutionException( - "EXE0001", - ErrorLevel.ERROR, - CommonErrMsg.ExecutionInitErr, - "Multiple Jobs is not Supported by current execution"); - } - - Job job = jobs.get(jobs.keySet().toArray(new String[jobs.size()])[0]); - - if (!(job instanceof ManagableBackendJob)) { - throw new LinkisClientExecutionException( - "EXE0001", - ErrorLevel.ERROR, - CommonErrMsg.ExecutionInitErr, - "Backend for \"" + job.getClass().getCanonicalName() + "\" is not manageable"); - } - - if (job.getSubType() == null) { - throw new LinkisClientExecutionException( - "EXE0001", - ErrorLevel.ERROR, - CommonErrMsg.ExecutionInitErr, - "SubExecType should not be null"); - } - - try { - ((ManagableBackendJob) job).doManage(); - if (((ManagableBackendJob) job).isSuccess()) { - executionStatus = ExecutionStatusEnum.SUCCEED; - } else { - executionStatus = ExecutionStatusEnum.FAILED; - if (job.getJobData() != null && job.getJobData().getException() != null) { - exception = job.getJobData().getException(); - } - } - - } catch (Exception e) { - exception = e; - executionStatus = ExecutionStatusEnum.FAILED; - } - - return new ExecutionResultImpl(jobs, executionStatus, exception); - } - - @Override - public boolean terminate(Map jobs) { - return true; - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/execution/SyncSubmission.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/execution/SyncSubmission.java deleted file mode 100644 index c143e22357..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/execution/SyncSubmission.java +++ /dev/null @@ -1,195 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.interactor.execution; - -import org.apache.linkis.cli.common.entity.execution.Execution; -import org.apache.linkis.cli.common.entity.job.Job; -import org.apache.linkis.cli.common.entity.result.ExecutionResult; -import org.apache.linkis.cli.common.entity.result.ExecutionStatus; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.constants.CommonConstants; -import org.apache.linkis.cli.core.exception.LinkisClientExecutionException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; -import org.apache.linkis.cli.core.interactor.job.*; -import org.apache.linkis.cli.core.interactor.result.ExecutionResultImpl; -import org.apache.linkis.cli.core.interactor.result.ExecutionStatusEnum; -import org.apache.linkis.cli.core.utils.CommonUtils; -import org.apache.linkis.cli.core.utils.LogUtils; - -import org.apache.commons.lang3.exception.ExceptionUtils; - -import java.util.Map; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Execute job synchronously. i.e. Client submit job, and wait til job finish, and get result, no - * matter what server behaves. - */ -public class SyncSubmission implements Execution { - private static final Logger logger = LoggerFactory.getLogger(SyncSubmission.class); - - @Override - public ExecutionResult execute(Map jobs) { - - ExecutionStatus executionStatus; - Exception exception = null; // TODO - - if (jobs == null || jobs.size() == 0) { - throw new LinkisClientExecutionException( - "EXE0001", - ErrorLevel.ERROR, - CommonErrMsg.ExecutionErr, - "Null or empty Jobs is submitted to current execution"); - } - - if (jobs.size() > 1) { - throw new LinkisClientExecutionException( - "EXE0001", - ErrorLevel.ERROR, - CommonErrMsg.ExecutionErr, - "Multiple Jobs is not Supported by current execution"); - } - - Job job = jobs.get(jobs.keySet().toArray(new String[jobs.size()])[0]); - - if (job instanceof SyncBackendJob) { - try { - ((SyncBackendJob) job).submitAndGetResult(); - } catch (Exception e) { - exception = e; - } - } else if (job instanceof AsyncBackendJob) { - try { - ExecWithAsyncBackend(job); - } catch (Exception e) { - exception = e; - // TODO: throw or fail - } - } else { - throw new LinkisClientExecutionException( - "EXE0002", - ErrorLevel.ERROR, - CommonErrMsg.ExecutionErr, - "Executor Type: \"" + job.getClass().getCanonicalName() + "\" is not Supported"); - } - - if (job.getJobData() != null - && job.getJobData().getJobStatus() != null - && job.getJobData().getJobStatus().isJobSuccess()) { - executionStatus = ExecutionStatusEnum.SUCCEED; - } else if (job.getJobData().getJobStatus() == null - || !job.getJobData().getJobStatus().isJobFinishedState()) { - executionStatus = ExecutionStatusEnum.UNDEFINED; - if (job.getJobData().getException() != null) { - exception = job.getJobData().getException(); - } - } else { - executionStatus = ExecutionStatusEnum.FAILED; - if (job.getJobData().getException() != null) { - exception = job.getJobData().getException(); - } - } - - return new ExecutionResultImpl(jobs, executionStatus, exception); - } - - @Override - public boolean terminate(Map jobs) { - boolean ok = true; - for (Job job : jobs.values()) { - if (job.getJobData() == null || job.getJobData().getJobStatus() == null) { - continue; - } - String jobId = job.getJobData().getJobID() == null ? "NULL" : job.getJobData().getJobID(); - if (job instanceof TerminatableJob) { - try { - ((TerminatableJob) job).terminate(); - } catch (Exception e) { - System.out.println( - "Failed to kill job: jobId=" + jobId + ". " + ExceptionUtils.getStackTrace(e)); - } - if (!job.getJobData().getJobStatus().isJobCancelled() - || !job.getJobData().getJobStatus().isJobFailure()) { - ok = false; - System.out.println( - "Failed to kill job: jobId=" - + jobId - + ", current status: " - + job.getJobData().getJobStatus().toString()); - } else { - System.out.println( - "Successfully killed job: jobId=" - + jobId - + ", current status: " - + job.getJobData().getJobStatus().toString()); - } - } else { - System.out.println("Job \"" + jobId + "\"" + "is not terminatable"); - } - } - return ok; - } - - private void ExecWithAsyncBackend(Job job) { - - if (!(job instanceof AsyncBackendJob)) { - throw new LinkisClientExecutionException( - "EXE0002", - ErrorLevel.ERROR, - CommonErrMsg.ExecutionErr, - "job is not instance of AsyncBackendJob"); - } - AsyncBackendJob submitJob = (AsyncBackendJob) job; - - submitJob.submit(); - CommonUtils.doSleepQuietly(CommonConstants.JOB_QUERY_SLEEP_MILLS); - - if (!submitJob.getJobData().getJobStatus().isJobSubmitted()) { - throw new LinkisClientExecutionException( - "EXE0005", - ErrorLevel.ERROR, - CommonErrMsg.ExecutionErr, - "Retry exhausted checking job submission. Job is probably not submitted"); - } else { - // Output that job is submitted - StringBuilder infoBuilder = new StringBuilder(); - infoBuilder.append("Job is successfully submitted!").append(System.lineSeparator()); - LogUtils.getInformationLogger().info(infoBuilder.toString()); - } - - if (job instanceof LogAccessibleJob) { - /* - Non-blocking, call if back-end supports it - */ - ((LogAccessibleJob) job).startRetrieveLog(); - } - - submitJob.waitJobComplete(); - - if (submitJob.getJobData().getJobStatus().isJobFinishedState()) { - if (job instanceof ResultAccessibleJob) { - /* - Non-blocking, call if back-end supports it - */ - ((ResultAccessibleJob) job).startRetrieveResult(); - } - } - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/job/AbstractJob.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/job/AbstractJob.java deleted file mode 100644 index 0de5e27ce6..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/job/AbstractJob.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.interactor.job; - -import org.apache.linkis.cli.common.entity.command.CmdType; -import org.apache.linkis.cli.common.entity.job.Job; -import org.apache.linkis.cli.common.entity.job.JobSubType; -import org.apache.linkis.cli.common.entity.operator.JobOperator; -import org.apache.linkis.cli.common.entity.present.PresentWay; - -public abstract class AbstractJob implements Job { - protected String cid; - protected CmdType cmdType; - protected JobSubType subType; - protected JobOperator operator; - private PresentWay presentWay; - - @Override - public String getCid() { - return cid; - } - - public void setCid(String cid) { - this.cid = cid; - } - - @Override - public CmdType getCmdType() { - return cmdType; - } - - public void setCmdType(CmdType cmdType) { - this.cmdType = cmdType; - } - - @Override - public JobSubType getSubType() { - return subType; - } - - public void setSubType(JobSubType subType) { - this.subType = subType; - } - - @Override - public JobOperator getJobOperator() { - return operator; - } - - public void setOperator(JobOperator operator) { - this.operator = operator; - } - - @Override - public PresentWay getPresentWay() { - return presentWay; - } - - public void setPresentWay(PresentWay presentWay) { - this.presentWay = presentWay; - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/job/AsyncBackendJob.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/job/AsyncBackendJob.java deleted file mode 100644 index 636a361cad..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/job/AsyncBackendJob.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.interactor.job; - -import org.apache.linkis.cli.common.entity.job.Job; -import org.apache.linkis.cli.common.entity.job.JobData; -import org.apache.linkis.cli.common.exception.LinkisClientRuntimeException; - -/** - * If backend supports async-submission, i.e. submit job and then return while job is running. Then - * implement this. Note that all results return by server should be returned but stored in {@link - * JobData} - */ -public interface AsyncBackendJob extends Job { - - void submit() throws LinkisClientRuntimeException; - - void updateJobStatus() throws LinkisClientRuntimeException; - - void waitJobComplete() throws LinkisClientRuntimeException; - - void terminate() throws LinkisClientRuntimeException; -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/job/JobBuilder.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/job/JobBuilder.java deleted file mode 100644 index 566fe3b279..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/job/JobBuilder.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.interactor.job; - -import org.apache.linkis.cli.common.entity.command.CmdType; -import org.apache.linkis.cli.common.entity.job.JobData; -import org.apache.linkis.cli.common.entity.job.JobDescription; -import org.apache.linkis.cli.common.entity.job.JobSubType; -import org.apache.linkis.cli.common.entity.operator.JobOperator; -import org.apache.linkis.cli.common.entity.present.PresentWay; -import org.apache.linkis.cli.core.builder.BuildableByVarAccess; - -public abstract class JobBuilder extends BuildableByVarAccess { - - public JobBuilder setCid(String cid) { - targetObj.setCid(cid); - return this; - } - - public JobBuilder setCmdType(CmdType cmdType) { - targetObj.setCmdType(cmdType); - return this; - } - - public JobBuilder setJobSubType(JobSubType subType) { - targetObj.setSubType(subType); - return this; - } - - protected abstract JobDescription buildJobDesc(); - - protected abstract JobData buildJobData(); - - protected abstract JobOperator buildJobOperator(); - - protected abstract PresentWay buildPresentWay(); -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/job/LogAccessibleJob.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/job/LogAccessibleJob.java deleted file mode 100644 index d372c444a3..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/job/LogAccessibleJob.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.interactor.job; - -import org.apache.linkis.cli.common.entity.job.Job; -import org.apache.linkis.cli.common.entity.job.JobData; - -/** If backend supports extracting log while job is running, then implement this interface */ -public interface LogAccessibleJob extends Job { - /** - * Retrieve log. This methods should be non-blocking and log should be appended into {@link - * JobData} - */ - void startRetrieveLog(); -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/job/ManagableBackendJob.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/job/ManagableBackendJob.java deleted file mode 100644 index 204134110b..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/job/ManagableBackendJob.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.interactor.job; - -import org.apache.linkis.cli.common.entity.job.Job; -import org.apache.linkis.cli.common.entity.job.JobStatus; -import org.apache.linkis.cli.common.entity.job.JobSubType; -import org.apache.linkis.cli.common.exception.LinkisClientRuntimeException; - -/** - * Backend support job-management, e.g. get status, get log, kill job etc. Then implement this - * interface. - */ -public interface ManagableBackendJob extends Job { - /** - * since job management can diverge, we decide to assign it to lower-level implementation. - * implementation should use {@link JobSubType} to decide which action to take, hence {@link - * JobSubType} should not be null - */ - void doManage() throws LinkisClientRuntimeException; - - /** - * if execution is success. This can be different from {@link JobStatus} e.g. query job status, - * job may be FAIL but execution is a asuccess - */ - boolean isSuccess(); -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/job/TerminatableJob.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/job/TerminatableJob.java deleted file mode 100644 index ff7e832724..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/job/TerminatableJob.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.interactor.job; - -import org.apache.linkis.cli.common.entity.job.Job; -import org.apache.linkis.cli.common.exception.LinkisClientRuntimeException; - -/** Backend supports killing job */ -public interface TerminatableJob extends Job { - /** for jobs that starts a new thread */ - TerminateToken getTerminateToken(); - - void terminate() throws LinkisClientRuntimeException; -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/properties/PropertiesLoader.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/properties/PropertiesLoader.java deleted file mode 100644 index df79c8262e..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/properties/PropertiesLoader.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.interactor.properties; - -import org.apache.linkis.cli.common.entity.properties.ClientProperties; -import org.apache.linkis.cli.core.interactor.properties.reader.PropertiesReader; - -public interface PropertiesLoader { - PropertiesLoader setPropertiesReaders(PropertiesReader[] readers); - - PropertiesLoader addPropertiesReader(PropertiesReader reader); - - PropertiesLoader addPropertiesReaders(PropertiesReader[] readers); - - void removePropertiesReader(String identifier); - - ClientProperties[] loadProperties(); - - void checkInit(); -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/properties/reader/SysEnvReader.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/properties/reader/SysEnvReader.java deleted file mode 100644 index b89ebe9755..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/properties/reader/SysEnvReader.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.interactor.properties.reader; - -import org.apache.linkis.cli.core.constants.CommonConstants; - -import java.util.Properties; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class SysEnvReader implements PropertiesReader { - private static final Logger logger = LoggerFactory.getLogger(SysEnvReader.class); - private String propsId = CommonConstants.SYSTEM_ENV_IDENTIFIER; - private String propsPath = "SYSTEM"; - - @Override - public String getPropsId() { - return propsId; - } - - @Override - public PropertiesReader setPropsId(String identifier) { - this.propsId = identifier; - return this; - } - - @Override - public String getPropsPath() { - return propsPath; - } - - @Override - public PropertiesReader setPropsPath(String propertiesPath) { - this.propsPath = propertiesPath; - return this; - } - - @Override - public Properties getProperties() { - checkInit(); - Properties props = new Properties(); - props.putAll(System.getenv()); - return props; - } - - @Override - public void checkInit() {} -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/properties/reader/SysPropsReader.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/properties/reader/SysPropsReader.java deleted file mode 100644 index bf4bbc0270..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/properties/reader/SysPropsReader.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.interactor.properties.reader; - -import org.apache.linkis.cli.core.constants.CommonConstants; - -import java.util.Properties; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class SysPropsReader implements PropertiesReader { - private static final Logger logger = LoggerFactory.getLogger(SysPropsReader.class); - private String propsId = CommonConstants.SYSTEM_PROPERTIES_IDENTIFIER; - private String propsPath = "SYSTEM"; - - @Override - public String getPropsId() { - return propsId; - } - - @Override - public PropertiesReader setPropsId(String identifier) { - this.propsId = identifier; - return this; - } - - @Override - public String getPropsPath() { - return propsPath; - } - - @Override - public PropertiesReader setPropsPath(String propertiesPath) { - this.propsPath = propertiesPath; - return this; - } - - @Override - public Properties getProperties() { - checkInit(); - return System.getProperties(); - } - - @Override - public void checkInit() {} -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/result/DefaultResultHandler.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/result/DefaultResultHandler.java deleted file mode 100644 index 836aeb7ad8..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/result/DefaultResultHandler.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.interactor.result; - -import org.apache.linkis.cli.common.entity.result.ExecutionResult; -import org.apache.linkis.cli.common.entity.result.ResultHandler; -import org.apache.linkis.cli.core.constants.CommonConstants; -import org.apache.linkis.cli.core.utils.LogUtils; - -import static java.lang.System.exit; - -/** exit -1 when failure and exit 0 when success */ -public class DefaultResultHandler implements ResultHandler { - @Override - public void process(ExecutionResult executionResult) { - if (executionResult.getExecutionStatus() == ExecutionStatusEnum.SUCCEED) { - LogUtils.getPlaintTextLogger().info(CommonConstants.SUCCESS_INDICATOR); - exit(0); - } else { - LogUtils.getPlaintTextLogger().info(CommonConstants.FAILURE_INDICATOR); - exit(-1); - } - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/result/ExecutionResultImpl.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/result/ExecutionResultImpl.java deleted file mode 100644 index b145ba9c8b..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/result/ExecutionResultImpl.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.interactor.result; - -import org.apache.linkis.cli.common.entity.job.Job; -import org.apache.linkis.cli.common.entity.result.ExecutionResult; -import org.apache.linkis.cli.common.entity.result.ExecutionStatus; - -import java.util.Map; - -public class ExecutionResultImpl implements ExecutionResult { - - Map jobsMap; - ExecutionStatus executionStatus; - Exception exception; - - public ExecutionResultImpl(Map jobsMap, ExecutionStatus executionStatus) { - this.jobsMap = jobsMap; - this.executionStatus = executionStatus; - } - - public ExecutionResultImpl( - Map jobsMap, ExecutionStatus executionStatus, Exception exception) { - this.jobsMap = jobsMap; - this.executionStatus = executionStatus; - this.exception = exception; - } - - @Override - public Map getJobs() { - return jobsMap; - } - - @Override - public ExecutionStatus getExecutionStatus() { - return this.executionStatus; - } - - @Override - public void setExecutionStatus(ExecutionStatus executionStatus) { - this.executionStatus = executionStatus; - } - - @Override - public Exception getException() { - return this.exception; - } - - @Override - public void setException(Exception exception) { - this.exception = exception; - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/result/PresentResultHandler.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/result/PresentResultHandler.java deleted file mode 100644 index 5a3fbfa4a2..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/result/PresentResultHandler.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.interactor.result; - -import org.apache.linkis.cli.common.entity.job.Job; -import org.apache.linkis.cli.common.entity.present.Model; -import org.apache.linkis.cli.common.entity.present.Presenter; -import org.apache.linkis.cli.common.entity.result.ExecutionResult; -import org.apache.linkis.cli.common.entity.result.ResultHandler; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.exception.LinkisClientExecutionException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; - -import java.util.Map; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class PresentResultHandler implements ResultHandler { - private static Logger logger = LoggerFactory.getLogger(PresentResultHandler.class); - Presenter presenter; - Model model; - - public void checkInit() { - if (presenter == null || model == null) { - throw new LinkisClientExecutionException( - "EXE0031", - ErrorLevel.ERROR, - CommonErrMsg.ExecutionResultErr, - "Presenter or model is null"); - } - } - - public void setPresenter(Presenter presenter) { - this.presenter = presenter; - } - - public void setModel(Model model) { - this.model = model; - } - - @Override - public void process(ExecutionResult executionResult) { - checkInit(); - Map jobs = executionResult.getJobs(); - // Probably need modification if we further want multiple-jobs support - // but we probably don't want to support that - if (jobs != null) { - for (Job job : jobs.values()) { - if (job != null) { - model.buildModel(job.getJobData()); - } - try { - presenter.present(model, job.getPresentWay()); - } catch (Exception e) { - logger.error("Execution failed because exception thrown when presenting data.", e); - executionResult.setExecutionStatus(ExecutionStatusEnum.FAILED); - executionResult.setException(e); - } - } - } - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/var/SysVarAccess.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/var/SysVarAccess.java deleted file mode 100644 index d291e1bda5..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/var/SysVarAccess.java +++ /dev/null @@ -1,114 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.interactor.var; - -import org.apache.linkis.cli.common.entity.properties.ClientProperties; -import org.apache.linkis.cli.common.entity.var.VarAccess; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.exception.VarAccessException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; - -import java.util.ArrayList; -import java.util.List; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class SysVarAccess implements VarAccess { - private static Logger logger = LoggerFactory.getLogger(SysVarAccess.class); - private ClientProperties sysProp; - private ClientProperties sysEnv; - - public SysVarAccess setSysProp(ClientProperties sysProp) { - this.sysProp = sysProp; - return this; - } - - public ClientProperties getSysProp(String identifier) { - return this.sysProp; - } - - public SysVarAccess setSysEnv(ClientProperties sysEnv) { - this.sysEnv = sysEnv; - return this; - } - - public ClientProperties getSysEnv(String identifier) { - return this.sysEnv; - } - - @Override - public void checkInit() { - if (this.sysProp == null && this.sysEnv == null) { - throw new VarAccessException( - "VA0001", - ErrorLevel.ERROR, - CommonErrMsg.VarAccessInitErr, - "sys_prop and sys_env are both null"); - } - } - - @Override - public T getVar(Class clazz, String key) { - checkInit(); - if (clazz != String.class) { - // throw exception - } - Object o1 = sysProp.get(key); - Object o2 = sysEnv.get(key); - if (o1 != null && o2 != null) { - throw new VarAccessException( - "VA0002", - ErrorLevel.WARN, - CommonErrMsg.VarAccessErr, - "same key occurred in sys_prop and sys_env. will use sys_prop"); - } - Object ret = o1 != null ? o1 : o2; - return clazz.cast(ret); - } - - @Override - public T getVarOrDefault(Class clazz, String key, T defaultValue) { - T ret = getVar(clazz, key); - if (ret == null) { - ret = defaultValue; - } - return ret; - } - - @Override - public String[] getAllVarKeys() { - List varKeys = new ArrayList<>(); - if (sysProp != null) { - for (Object key : sysProp.keySet()) { - varKeys.add((String) key); - } - } - if (sysEnv != null) { - for (Object key : sysEnv.keySet()) { - varKeys.add((String) key); - } - } - return varKeys.toArray(new String[varKeys.size()]); - } - - @Override - public boolean hasVar(String key) { - return sysEnv.containsKey(key) || sysProp.containsKey(key); - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/operator/JobOperatorFactory.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/operator/JobOperatorFactory.java deleted file mode 100644 index dd6031d66a..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/operator/JobOperatorFactory.java +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.operator; - -import org.apache.linkis.cli.common.entity.operator.JobOperator; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.exception.LinkisClientExecutionException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; - -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.locks.ReentrantLock; - -public class JobOperatorFactory { - private static Map builderMap = new ConcurrentHashMap<>(); - - private static Map instanceMap = new ConcurrentHashMap<>(); // for singleton - private static Map lockMap = new ConcurrentHashMap<>(); // for singleton - - public static synchronized void register(String name, JobOperatorBuilder builder) - throws Exception { - if (builderMap.containsKey(name) - || lockMap.containsKey(name) - || instanceMap.containsKey(name)) { - throw new LinkisClientExecutionException( - "EXE0027", - ErrorLevel.ERROR, - CommonErrMsg.ExecutionInitErr, - "Attempting to register a duplicate jobOperator, name: " + name); - } - builderMap.put(name, builder); - lockMap.put(name, new ReentrantLock()); - } - - public static synchronized void remove(String name) { - builderMap.remove(name); - instanceMap.remove(name); - lockMap.remove(name); - } - - public static JobOperator getReusable(String name) throws Exception { - JobOperatorBuilder builder = builderMap.get(name); - ReentrantLock lock = lockMap.get(name); - JobOperator instance = instanceMap.get(name); - if (lock == null || builder == null) { - throw new LinkisClientExecutionException( - "EXE0028", - ErrorLevel.ERROR, - CommonErrMsg.ExecutionInitErr, - "Failed to get a reusable joboperator, name: " + name); - } - if (instance == null) { - boolean ok = lock.tryLock(500, TimeUnit.MILLISECONDS); - if (!ok) { - throw new LinkisClientExecutionException( - "EXE0028", - ErrorLevel.ERROR, - CommonErrMsg.ExecutionInitErr, - "Failed to get a reusable joboperator, name: " + name); - } - if (instance == null) { - instance = builder.build(); - instanceMap.put(name, instance); - } - lock.unlock(); - } - return instance; - } - - public static JobOperator getNew(String name) throws Exception { - JobOperatorBuilder builder = builderMap.get(name); - if (builder == null) { - throw new Exception("TODO"); // TODO - } - return builder.build(); - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/present/DefaultStdOutPresenter.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/present/DefaultStdOutPresenter.java deleted file mode 100644 index 7ed2307520..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/present/DefaultStdOutPresenter.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.present; - -import org.apache.linkis.cli.common.entity.present.Model; -import org.apache.linkis.cli.common.entity.present.PresentWay; -import org.apache.linkis.cli.common.entity.present.Presenter; -import org.apache.linkis.cli.core.present.display.DisplayOperator; -import org.apache.linkis.cli.core.present.display.StdOutWriter; -import org.apache.linkis.cli.core.present.display.data.StdoutDisplayData; -import org.apache.linkis.cli.core.utils.CommonUtils; - -public class DefaultStdOutPresenter implements Presenter { - DisplayOperator driver = new StdOutWriter(); - - @Override - public void present(Model model, PresentWay presentWay) { - driver.doOutput(new StdoutDisplayData(CommonUtils.GSON.toJson(model))); - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/present/PresentMode.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/present/PresentMode.java deleted file mode 100644 index 6d26a2eb49..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/present/PresentMode.java +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.present; - -public interface PresentMode { - String getName(); -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/present/display/DisplayOperFactory.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/present/display/DisplayOperFactory.java deleted file mode 100644 index 9f584f4505..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/present/display/DisplayOperFactory.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.present.display; - -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.exception.PresenterException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; -import org.apache.linkis.cli.core.present.PresentMode; - -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - -public class DisplayOperFactory { - private static final Map operatorMap = new ConcurrentHashMap<>(); - - public static synchronized void register(PresentMode mode, DisplayOperator operator) - throws Exception { - if (operatorMap.containsKey(mode.getName())) { - throw new PresenterException( - "PST0012", - ErrorLevel.ERROR, - CommonErrMsg.PresenterInitErr, - "Attempting to register a duplicate DisplayOperator, name: " + mode.getName()); - } - operatorMap.put(mode.getName(), operator); - } - - public static synchronized void remove(PresentMode mode) { - operatorMap.remove(mode.getName()); - } - - public static DisplayOperator getDisplayOper(PresentMode mode) { - return operatorMap.get(mode.getName()); - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/present/display/StdOutWriter.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/present/display/StdOutWriter.java deleted file mode 100644 index bc91340dbf..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/present/display/StdOutWriter.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.present.display; - -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.exception.PresenterException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; -import org.apache.linkis.cli.core.present.display.data.DisplayData; -import org.apache.linkis.cli.core.present.display.data.StdoutDisplayData; -import org.apache.linkis.cli.core.utils.LogUtils; - -import org.slf4j.Logger; - -public class StdOutWriter implements DisplayOperator { - @Override - public void doOutput(DisplayData data) { - if (!(data instanceof StdoutDisplayData)) { - throw new PresenterException( - "PST0008", - ErrorLevel.ERROR, - CommonErrMsg.PresentDriverErr, - "input data is not instance of StdoutDisplayData"); - } - String content = ((StdoutDisplayData) data).getContent(); - Logger logger = LogUtils.getPlaintTextLogger(); - logger.info(content); - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/present/display/data/DisplayData.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/present/display/data/DisplayData.java deleted file mode 100644 index 38463b7269..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/present/display/data/DisplayData.java +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.present.display.data; - -public interface DisplayData {} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/present/display/data/FileDisplayData.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/present/display/data/FileDisplayData.java deleted file mode 100644 index 607e05fd86..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/present/display/data/FileDisplayData.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.present.display.data; - -public class FileDisplayData extends StdoutDisplayData implements DisplayData { - private String pathName; - private String fileName; - private Boolean createNewFile; - - public FileDisplayData(String pathName, String fileName, String content, Boolean createNewFile) { - super(content); - this.pathName = pathName; - this.fileName = fileName; - this.createNewFile = createNewFile; - } - - public String getPathName() { - return pathName; - } - - public String getContent() { - return super.getContent(); - } - - public boolean getCreateNewFile() { - return createNewFile; - } - - public String getFileName() { - return fileName; - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/utils/CommonUtils.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/utils/CommonUtils.java deleted file mode 100644 index 8d1a3b8ac9..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/utils/CommonUtils.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.utils; - -import org.apache.commons.lang3.StringUtils; - -import java.util.HashMap; -import java.util.Map; - -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; - -public class CommonUtils { - - public static final Gson GSON = - new GsonBuilder().setPrettyPrinting().disableHtmlEscaping().create(); - - public static T castStringToAny(Class clazz, String val) { - if (StringUtils.isBlank(val)) { - return null; - } - T ret = null; - if (clazz == Object.class) { - ret = clazz.cast(val); - } else if (clazz == String.class) { - ret = clazz.cast(val); - } else if (clazz == Integer.class) { - ret = clazz.cast(Integer.parseInt(val)); - } else if (clazz == Double.class) { - ret = clazz.cast(Double.parseDouble(val)); - } else if (clazz == Float.class) { - ret = clazz.cast(Float.parseFloat(val)); - } else if (clazz == Long.class) { - ret = clazz.cast(Long.parseLong(val)); - } else if (clazz == Boolean.class) { - ret = clazz.cast(Boolean.parseBoolean(val)); - } - return ret; - } - - public static void doSleepQuietly(Long sleepMills) { - try { - Thread.sleep(sleepMills); - } catch (Exception ignore) { - // ignored - } - } - - public static Map parseKVStringToMap(String kvStr, String separator) { - if (StringUtils.isBlank(separator)) { - separator = ","; - } - if (StringUtils.isBlank(kvStr)) { - return null; - } - Map argsProps = new HashMap<>(); - String[] args = StringUtils.splitByWholeSeparator(kvStr, separator); - for (String arg : args) { - int index = arg.indexOf("="); - if (index != -1) { - argsProps.put(arg.substring(0, index).trim(), arg.substring(index + 1).trim()); - } - } - - return argsProps; - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/constants/CommonConstantsTest.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/constants/CommonConstantsTest.java deleted file mode 100644 index a1fb04f01d..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/constants/CommonConstantsTest.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.constants; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.Test; - -public class CommonConstantsTest { - - @Test - @DisplayName("constTest") - public void constTest() { - - Long jobQuerySleepMills = CommonConstants.JOB_QUERY_SLEEP_MILLS; - Integer requestMaxRetryTime = CommonConstants.REQUEST_MAX_RETRY_TIME; - String universalSubcmd = CommonConstants.UNIVERSAL_SUBCMD; - String universalSubcmdDesc = CommonConstants.UNIVERSAL_SUBCMD_DESC; - String successIndicator = CommonConstants.SUCCESS_INDICATOR; - String failureIndicator = CommonConstants.FAILURE_INDICATOR; - String arraySeq = CommonConstants.ARRAY_SEQ; - String arraySeqRegex = CommonConstants.ARRAY_SEQ_REGEX; - int maxNumOfCommandArguements = CommonConstants.MAX_NUM_OF_COMMAND_ARGUEMENTS; - String configDir = CommonConstants.CONFIG_DIR; - String[] configExtension = CommonConstants.CONFIG_EXTENSION; - String systemPropertiesIdentifier = CommonConstants.SYSTEM_PROPERTIES_IDENTIFIER; - String systemEnvIdentifier = CommonConstants.SYSTEM_ENV_IDENTIFIER; - - Assertions.assertTrue(2000l == jobQuerySleepMills); - Assertions.assertTrue(3 == requestMaxRetryTime); - Assertions.assertEquals("linkis-cli", universalSubcmd); - Assertions.assertEquals( - "command for all types of jobs supported by Linkis", universalSubcmdDesc); - - Assertions.assertEquals("############Execute Success!!!########", successIndicator); - Assertions.assertEquals("############Execute Error!!!########", failureIndicator); - Assertions.assertEquals("@#@", arraySeq); - Assertions.assertEquals("(?=([^\"]*\"[^\"]*\")*[^\"]*$)", arraySeqRegex); - Assertions.assertTrue(10 == maxNumOfCommandArguements); - Assertions.assertEquals("config.path", configDir); - Assertions.assertTrue(configExtension.length == 1); - Assertions.assertEquals("SYS_PROP", systemPropertiesIdentifier); - Assertions.assertEquals("SYS_ENV", systemEnvIdentifier); - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/constants/TestConstants.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/constants/TestConstants.java deleted file mode 100644 index 307369054b..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/constants/TestConstants.java +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.constants; - -public class TestConstants { - - public static final String PRIMARY = "test.primary"; - public static final String PRIMARY_DESC = " command"; - - // for command name - public static final String SPARK = "spark"; - public static final String HIVE = "hive"; - - // for command description - public static final String SPARK_DESC = "Execute sql with spark"; - public static final String HIVE_DESC = "Execute hive sql"; - public static final String JOB_DESC = ""; - - // Common command params - public static final String PARAM_COMMON_FILE = "param.common.file"; - public static final String PARAM_COMMON_CMD = "param.common.command"; - public static final String PARAM_COMMON_OUTPUT = "param.common.output"; - public static final String PARAM_COMMON_ARGS = "param.common.args"; - public static final String PARAM_COMMON_SPLIT = "param.common.split"; - public static final String PARAM_COMMON_OTHER_KV = - "param.common.other.kv"; // for customize some parameters for some commands. Has to be - // in the for of kv-pairs separated by ',' - public static final String PARAM_YARN_QUEUE = "param.yarn.queue"; - - // for job command - public static final String PARAM_JOB_TYPE = "param.job.type"; - public static final String PARAM_JOB_ID = "param.job.id"; - public static final String PARAM_FORCE_KILL = "param.force.kill"; - public static final String LINKIS_JOBID_PREFIX = "lks_"; - - public static final String PARAM_MAPPER_KV_STRING = - "param.mapper.kv.string"; // Rules for ParamKeyMapper. format: - // key1=value1,key2=value2... - public static final String YARN_QUEUE = "wds.linkis.yarnqueue"; - public static final String YARN_QUEUE_DEFAULT = "default"; - public static final String YARN_QUEUE_CORE_MAX = "wds.linkis.yarnqueue.cores.max"; - public static final int YARN_QUEUE_CORE_MAX_DEFAULT = 150; - public static final String YARN_QUEUE_MEM_MAX = "wds.linkis.yarnqueue.memory.max"; - public static final String YARN_QUEUE_MEM_MAX_DEFAULT = "300G"; - public static final String PREHEATING_TIME = "wds.linkis.preheating.time"; - public static final String PREHEATING_TIME_DEFAULT = "9:00"; - public static final String TMPFILE_CLEAN_TIME = "wds.linkis.tmpfile.clean.time"; - public static final String TMPFILE_CLEAN_TIME_DEFAULT = "10:00"; - public static final String LINKIS_INSTANCE = "wds.linkis.instance"; - public static final int LINKIS_INSTANCE_DEFAULT = 10; - public static final String LINKIS_CLIENT_MEMORY_MAX = "wds.linkis.client.memory.max"; - public static final String LINKIS_CLIENT_MEMORY_MAX_DEFAULT = "20G"; - - // Common - public static final String LINKIS_NULL_VALUE = ""; - - public static final String SPARK_CMD = "spark"; - - public static final String PARAM_SPARK_NAME = "param.spark.name"; - public static final String PARAM_SPARK_HIVECONF = "param.spark.hiveconf"; - public static final String PARAM_SPARK_NUM_EXECUTORS = "param.spark.num.executors"; - public static final String PARAM_SPARK_EXECUTOR_CORES = "param.spark.executor.cores"; - public static final String PARAM_SPARK_EXECUTOR_MEMORY = "param.spark.executor.memory"; - public static final String PARAM_SPARK_SHUFFLE_PARTITIONS = "param.spark.shuffle.partitions"; - public static final String PARAM_SPARK_RUNTYPE = "param.spark.runtype"; - - public static final String LINKIS_SPARK_NAME = "appName"; - public static final String LINKIS_SPARK_NUM_EXECUTORS = "spark.executor.instances"; - public static final String LINKIS_SPARK_EXECUTOR_CORES = "spark.executor.cores"; - public static final String LINKIS_SPARK_EXECUTOR_MEMORY = "spark.executor.memory"; - public static final String LINKIS_SPARK_SHUFFLE_PARTITIONS = "spark.sql.shuffle.partitions"; - - public static final String PARAM_DB = "param.primary.database"; - public static final String PARAM_PROXY = "param.primary.proxy"; - public static final String PARAM_USER = "param.primary.user"; - public static final String PARAM_USR_CONF = "param.primary.user.conf"; - public static final String PARAM_PASSWORD = "param.primary.password"; - public static final String PARAM_SYNC_KEY = "param.primary.synckey"; - public static final String PARAM_PROXY_USER = "param.primary.proxyUser"; - public static final String PARAM_HELP = "param.help"; - public static final String PARAM_REAL_NAME = "param.primary.realName"; - public static final String PARAM_PIN_TOKEN = "param.primary.pinToken"; - - public static final String PARAM_PROPERTIES = "params.properties"; -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/exception/error/CommonErrMsgTest.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/exception/error/CommonErrMsgTest.java deleted file mode 100644 index cdd51f4aea..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/exception/error/CommonErrMsgTest.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.exception.error; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.Test; - -public class CommonErrMsgTest { - - @Test - @DisplayName("enumTest") - public void enumTest() { - - String templateGenerr = CommonErrMsg.TemplateGenErr.getMsgTemplate(); - String fitErrMsgTemplate = CommonErrMsg.TemplateFitErr.getMsgTemplate(); - String parserInitErrMsgTemplate = CommonErrMsg.ParserInitErr.getMsgTemplate(); - String parseErrMsgTemplate = CommonErrMsg.ParserParseErr.getMsgTemplate(); - - Assertions.assertEquals("Cannot generate template. :{0}", templateGenerr); - Assertions.assertEquals("Cannot fit input into template: {0}", fitErrMsgTemplate); - - Assertions.assertEquals("Failed to init parser: {0}", parserInitErrMsgTemplate); - Assertions.assertEquals("Failed to parse. {0}", parseErrMsgTemplate); - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/exception/handler/CommandExceptionHandlerTest.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/exception/handler/CommandExceptionHandlerTest.java deleted file mode 100644 index ee23f0216b..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/exception/handler/CommandExceptionHandlerTest.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.exception.handler; - -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.common.exception.handler.ExceptionHandler; -import org.apache.linkis.cli.core.exception.CommandException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; -import org.apache.linkis.cli.core.interactor.command.TestCmdType; - -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Test; - -import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; - -public class CommandExceptionHandlerTest { - ExceptionHandler handler = new CommandExceptionHandler(); - - // todo - @Disabled - @Test - public void handle() throws Exception { - CommandException cmdException = - new CommandException( - "CODE-001", - ErrorLevel.ERROR, - CommonErrMsg.TemplateGenErr, - "Failed to generate template."); - assertDoesNotThrow(() -> handler.handle(cmdException)); - - String[] params = {"param1", "param2"}; - CommandException cmdException2 = - new CommandException( - "CODE-001", - ErrorLevel.ERROR, - CommonErrMsg.TemplateGenErr, - params, - "Failed to generate template."); - assertDoesNotThrow(() -> handler.handle(cmdException2)); - - CommandException cmdException3 = - new CommandException( - "CODE-001", - ErrorLevel.ERROR, - CommonErrMsg.TemplateGenErr, - TestCmdType.PRIMARY, - "Failed to generate template."); - assertDoesNotThrow(() -> handler.handle(cmdException3)); - - CommandException cmdException4 = - new CommandException( - "CODE-001", - ErrorLevel.ERROR, - CommonErrMsg.TemplateGenErr, - TestCmdType.PRIMARY, - params, - "Failed to generate template."); - assertDoesNotThrow(() -> handler.handle(cmdException4)); - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/interactor/command/TestCmdType.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/interactor/command/TestCmdType.java deleted file mode 100644 index a6b05eef2b..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/interactor/command/TestCmdType.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.interactor.command; - -import org.apache.linkis.cli.common.entity.command.CmdType; -import org.apache.linkis.cli.core.constants.TestConstants; - -public enum TestCmdType implements CmdType { - PRIMARY(TestConstants.PRIMARY, 0, TestConstants.PRIMARY_DESC), - SPARK(TestConstants.SPARK, 1, TestConstants.SPARK_DESC); - // TEST(TestConstants.TEST_COMMAND, 0, TestConstants.TEST_DESC); - - private int id; - private String name; - private String desc; - - TestCmdType(String name, int id) { - this.id = id; - this.name = name; - this.desc = null; - } - - TestCmdType(String name, int id, String desc) { - this.id = id; - this.name = name; - this.desc = desc; - } - - @Override - public int getId() { - return this.id; - } - - @Override - public String getName() { - return this.name; - } - - @Override - public String getDesc() { - return this.desc; - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/interactor/command/fitter/FitterUtilsTest.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/interactor/command/fitter/FitterUtilsTest.java deleted file mode 100644 index 5c51a582fa..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/interactor/command/fitter/FitterUtilsTest.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.interactor.command.fitter; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.Test; - -public class FitterUtilsTest { - - @Test - @DisplayName("isOptionTest") - public void isOptionTest() { - - String name = "-hadoop"; - boolean option = FitterUtils.isOption(name); - Assertions.assertTrue(option); - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/interactor/command/fitter/SingleTplFitterTest.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/interactor/command/fitter/SingleTplFitterTest.java deleted file mode 100644 index ef542910dd..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/interactor/command/fitter/SingleTplFitterTest.java +++ /dev/null @@ -1,119 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.interactor.command.fitter; - -import org.apache.linkis.cli.common.entity.command.CmdTemplate; -import org.apache.linkis.cli.core.interactor.command.template.TestSparkCmdTemplate; - -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import static org.junit.jupiter.api.Assertions.*; - -public class SingleTplFitterTest { - Fitter fitter; - CmdTemplate template; - String[] cmdStr, cmdStr2; - - @BeforeEach - public void before() throws Exception { - cmdStr = - new String[] { - "-u", - "hadoop", - "-pwd", - "1234", - "-c", - "/path/to/user/config", - "--cmd", - "show tables", - "--split", - "\',\'", - "--queue", - "q05", - "--name", - "testApp", - // "--hiveconf", "/path/...", - "--num-executors", - "4", - "--executor-cores", - "4", - "--executor-memory", - "4G", - "--shuffle-partitions", - "200", - "--other", - "--other-spark-config=none", - }; - cmdStr2 = - new String[] { - "-u", - "hadoop", - "-pwd", - "1234", - "-c", - "/path/to/user/config", - "--cmd", - "show tables", - "--split", - "\',\'", - "--queue", - "q05", - "--name", - "testApp", - // "--hiveconf", "/path/...", - "--num-executors", - "4", - "--executor-cores", - "4", - "--executor-memory", - "4G", - "--shuffle-partitions", - "200", - "--other", - "--other-spark-config=none", - "-P", - "key1=value1, key2=value2, key5=\"key3=value3,key4=value4\" " - }; - template = new TestSparkCmdTemplate(); - fitter = new SingleTplFitter(); - } - - @AfterEach - public void after() throws Exception {} - - /** Method: fit(TemplateFitterInput[] inputs) */ - @Test - public void testParseAndFit() throws Exception { - FitterResult[] results = new FitterResult[2]; - results[0] = fitter.fit(cmdStr, template); - results[1] = fitter.fit(cmdStr2, new TestSparkCmdTemplate()); - - assertTrue(results[0].getParsedTemplate() instanceof TestSparkCmdTemplate); - assertEquals( - results[0].getParsedTemplate().getOptionsMap().get("--cmd").getValue(), "show tables"); - assertNotEquals(results[0].getParsedTemplate(), template.getCopy()); - assertNotEquals(results[0].getParsedTemplate().getOptions(), template.getCopy().getOptions()); - assertNotEquals( - results[0].getParsedTemplate().getOptions().get(1), template.getCopy().getOptions().get(1)); - assertEquals( - results[0].getParsedTemplate().getOptions().get(1).getValue(), - template.getCopy().getOptions().get(1).getValue()); - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/interactor/command/parser/SingleCmdParserTest.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/interactor/command/parser/SingleCmdParserTest.java deleted file mode 100644 index 61b8c72eab..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/interactor/command/parser/SingleCmdParserTest.java +++ /dev/null @@ -1,177 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.interactor.command.parser; - -import org.apache.linkis.cli.common.entity.command.CmdTemplate; -import org.apache.linkis.cli.common.entity.command.ParamItem; -import org.apache.linkis.cli.core.interactor.command.TestCmdType; -import org.apache.linkis.cli.core.interactor.command.fitter.SingleTplFitter; -import org.apache.linkis.cli.core.interactor.command.parser.result.ParseResult; -import org.apache.linkis.cli.core.interactor.command.template.TestParamMapper; -import org.apache.linkis.cli.core.interactor.command.template.TestSparkCmdTemplate; - -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import static org.junit.jupiter.api.Assertions.*; - -public class SingleCmdParserTest { - String[] cmdStr; - Map templateMap; - - @BeforeEach - public void before() throws Exception { - cmdStr = - new String[] { - "-u", - "hadoop", - "-pwd", - "1234", - "-c", - "/path/to/user/config", - "--cmd", - "show tables", - "--split", - "\',\'", - "--queue", - "q05", - "--name", - "testApp", - // "--hiveconf", "/path/...", - "--num-executors", - "4", - "--executor-cores", - "4", - "--executor-memory", - "4G", - "--shuffle-partitions", - "200", - "--other", - "--other-spark-config=none", - }; - - TestSparkCmdTemplate template = new TestSparkCmdTemplate(); - templateMap = new HashMap<>(); - templateMap.put(template.getCmdType().getName(), template); - } - - @AfterEach - public void after() throws Exception {} - - /** Method: parse(String[] input) */ - @Test - public void testParse() { - - Parser parser = - new SingleCmdParser() - .setMapper(null) - .setTemplate(templateMap.get(TestCmdType.SPARK.getName())) - .setFitter(new SingleTplFitter()) - .setMapper(new TestParamMapper()); - - ParseResult result = parser.parse(cmdStr); - - assertEquals(result.getParams().getCmdType(), TestCmdType.SPARK); - - Map params = result.getParams().getParamItemMap(); - - assertEquals(params.size(), 21); - - List sortedKeys = params.keySet().stream().sorted().collect(Collectors.toList()); - assertEquals( - sortedKeys, - Arrays.asList( - "converted.args", - "converted.split", - "param.common.command", - "param.common.file", - "param.common.other.kv", - "param.help", - "param.primary.database", - "param.primary.password", - "param.primary.proxy", - "param.primary.proxyUser", - "param.primary.synckey", - "param.primary.user", - "param.primary.user.conf", - "param.spark.executor.cores", - "param.spark.executor.memory", - "param.spark.hiveconf", - "param.spark.name", - "param.spark.num.executors", - "param.spark.runtype", - "param.spark.shuffle.partitions", - "param.yarn.queue")); - assertEquals(result.getRemains().length, 0); - } - - /** Method: parsePrimary(String[] input) */ - @Test - public void testParsePrimary() throws Exception { - // TODO: Test goes here... - /* - try { - Method method = SingleCmdParser.getClass().getMethod("parsePrimary", String[].class); - method.setAccessible(true); - method.invoke(, ); - } catch(NoSuchMethodException e) { - } catch(IllegalAccessException e) { - } catch(InvocationTargetException e) { - } - */ - } - - /** Method: parseSingleSub(String[] remains) */ - @Test - public void testParseSingleSub() throws Exception { - // TODO: Test goes here... - /* - try { - Method method = SingleCmdParser.getClass().getMethod("parseSingleSub", String[].class); - method.setAccessible(true); - method.invoke(, ); - } catch(NoSuchMethodException e) { - } catch(IllegalAccessException e) { - } catch(InvocationTargetException e) { - } - */ - } - - /** Method: standardParse(String identifier, String[] args, CmdTemplate templateOri) */ - @Test - public void testStandardParse() throws Exception { - // TODO: Test goes here... - /* - try { - Method method = SingleCmdParser.getClass().getMethod("standardParse", String.class, String[].class, CmdTemplate.class); - method.setAccessible(true); - method.invoke(, ); - } catch(NoSuchMethodException e) { - } catch(IllegalAccessException e) { - } catch(InvocationTargetException e) { - } - */ - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/interactor/command/template/TestParamMapper.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/interactor/command/template/TestParamMapper.java deleted file mode 100644 index 25f44d56c7..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/interactor/command/template/TestParamMapper.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.interactor.command.template; - -import org.apache.linkis.cli.core.constants.TestConstants; -import org.apache.linkis.cli.core.interactor.command.parser.transformer.ParamKeyMapper; - -public class TestParamMapper extends ParamKeyMapper { - - @Override - public void initMapperRules() { - super.updateMapping(TestConstants.SPARK, "converted.cmd"); - super.updateMapping(TestConstants.PARAM_COMMON_ARGS, "converted.args"); - super.updateMapping(TestConstants.PARAM_COMMON_SPLIT, "converted.split"); - // super.updateMapping("key1", "spark.cmd"); //should throw exception - // super.updateMapping("TestConstants.PARAM_SPARK_CMD", "spark.cmd"); - // super.updateMapping("TestConstants.PARAM_SPARK_CMD", "spark.cmd"); - - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/interactor/command/template/TestSparkCmdTemplate.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/interactor/command/template/TestSparkCmdTemplate.java deleted file mode 100644 index 4795f11c9c..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/interactor/command/template/TestSparkCmdTemplate.java +++ /dev/null @@ -1,268 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.interactor.command.template; - -import org.apache.linkis.cli.common.exception.LinkisClientRuntimeException; -import org.apache.linkis.cli.core.constants.TestConstants; -import org.apache.linkis.cli.core.exception.CommandException; -import org.apache.linkis.cli.core.interactor.command.TestCmdType; -import org.apache.linkis.cli.core.interactor.command.template.option.StdOption; - -import org.apache.commons.lang3.StringUtils; - -import java.util.HashMap; -import java.util.Map; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class TestSparkCmdTemplate extends AbstractCmdTemplate { - protected StdOption databaseOp = - option( - null, - TestConstants.PARAM_DB, - new String[] {"-d", "--database"}, - "specify database", - true, - ""); - protected StdOption proxyOp = - option( - null, - TestConstants.PARAM_PROXY, - new String[] {"-x", "--proxy"}, - "specify proxy url", - true, - ""); - protected StdOption userOp = - option( - null, TestConstants.PARAM_USER, new String[] {"-u", "--user"}, "specify user", true, ""); - protected StdOption confOp = - option( - null, - TestConstants.PARAM_USR_CONF, - new String[] {"-c", "--conf"}, - "specify configuration from property file", - true, - ""); - private Logger logger = LoggerFactory.getLogger(TestSparkCmdTemplate.class); - private StdOption passwordOp = - option( - null, - TestConstants.PARAM_PASSWORD, - new String[] {"-pwd", "--passwd"}, - "specify user password", - true, - ""); - private StdOption syncOp = - option( - null, - TestConstants.PARAM_SYNC_KEY, - new String[] {"-sk", "--synckey"}, - "specify sync key", - true, - ""); - private StdOption proxyUserOp = - option( - null, - TestConstants.PARAM_PROXY_USER, - new String[] {"-pu", "--proxy-user"}, - "specify proxy user", - true, - ""); - - private StdOption helpOp = - option(null, TestConstants.PARAM_HELP, new String[] {"-h", "--help"}, "help info", true, ""); - - private StdOption filePara = - option( - null, - TestConstants.PARAM_COMMON_FILE, - new String[] {"--file", "-f"}, - "Spark SQL File to Execute!", - true, - ""); - - private StdOption commandPara = - option( - null, - TestConstants.PARAM_COMMON_CMD, - new String[] {"--cmd"}, - "Spark SQL Command to Execute!", - true, - ""); - - private StdOption argsPara = - option( - null, - TestConstants.PARAM_COMMON_ARGS, - new String[] {"--args", "-a"}, - "Set command args, k-v pairs delimited by comma, e.g. key1=value1,key2=value2,...", - true, - ""); - - private StdOption splitPara = - option( - null, - TestConstants.PARAM_COMMON_SPLIT, - new String[] {"--split", "-s"}, - "specify the split character string", - true, - ","); - - private StdOption queuePara = - option( - null, - TestConstants.PARAM_YARN_QUEUE, - new String[] {"--queue", "-q"}, - "specify the queue", - true, - "default"); - - private StdOption namePara = - option( - null, - TestConstants.PARAM_SPARK_NAME, - new String[] {"--name", "-n"}, - "specify the application name. WARNING:this option is deprecated. Linkis does not support this variable", - true, - ""); - - private StdOption> hiveconfPara = - option( - null, - TestConstants.PARAM_SPARK_HIVECONF, - new String[] {"--hiveconf", "-hc"}, - "specify the hiveconf setting,e.g. hive.cli.print.header=false", - true, - new HashMap<>()); - - private StdOption nePara = - option( - null, - TestConstants.PARAM_SPARK_NUM_EXECUTORS, - new String[] {"--num-executors", "-ne"}, - "specify the spark application container", - true, - 3); - - private StdOption ecPara = - option( - null, - TestConstants.PARAM_SPARK_EXECUTOR_CORES, - new String[] {"--executor-cores", "-ec"}, - "specify the spark application container vcores(less than queue's max vcores)", - true, - 2); - - private StdOption emPara = - option( - null, - TestConstants.PARAM_SPARK_EXECUTOR_MEMORY, - new String[] {"--executor-memory", "-em"}, - "specify the spark application executor's memory, 1.5G-2G/vcore", - true, - "4G"); - - private StdOption spPara = - option( - null, - TestConstants.PARAM_SPARK_SHUFFLE_PARTITIONS, - new String[] {"--shuffle-partitions", "-sp"}, - "specify the spark.sql.shuffle.partitions", - true, - 200); - - private StdOption otherPara = - option( - null, - TestConstants.PARAM_COMMON_OTHER_KV, - new String[] {"--other"}, - "specify the other parameters", - true, - ""); - - private StdOption runTypePara = - option( - null, - TestConstants.PARAM_SPARK_RUNTYPE, - new String[] {"--runtype"}, - "specify the runtype parameters: sql pyspark scala", - true, - "sql"); - - public TestSparkCmdTemplate() { - super(TestCmdType.SPARK); - } - - @Override - public void checkParams() throws CommandException {} - - @Override - protected Object clone() throws CloneNotSupportedException { - return super.clone(); - } - - @Override - public TestSparkCmdTemplate getCopy() { - return (TestSparkCmdTemplate) super.getCopy(); - } - - public void prepare() throws LinkisClientRuntimeException {} - - private Map parseOtherMap(String otherStr) { - Map otherMap = new HashMap<>(); - /** - * split by space-chara, but if space char is within \"\", then do not split. e.g. "a b c" will - * not be splited - */ - otherStr = otherStr.replace("\'", "\""); - String[] arr = otherStr.trim().split("\\s(?=(?:[^\"]*\"[^\"]*\")*[^\"]*$)", -1); - for (int i = 0; i < arr.length; i++) { - String candidate = arr[i]; - - if (StringUtils.equalsIgnoreCase(candidate, "--conf") && i < arr.length - 1) { - // '--conf' and has next - i++; // move to next - candidate = arr[i]; - int idx = StringUtils.indexOf(candidate, "="); - if (idx != -1) { - String key = StringUtils.substring(candidate, 0, idx).trim(); - String value = StringUtils.substring(candidate, idx + 1).trim(); - otherMap.put(key, value); - } - } else { - // without '--conf' - int idx = StringUtils.indexOf(candidate, " "); - if (idx != -1) { - // e.g. '--driver-memory 2G' - String key = StringUtils.substring(candidate, 0, idx).trim(); - String value = StringUtils.substring(candidate, idx + 1).trim(); - otherMap.put(key, value); - } else if (i < arr.length - 1) { - // e.g. --driver-memory 2G - String key = candidate.trim(); - i++; // move to next - candidate = arr[i]; - String value = candidate.trim(); - otherMap.put(key, value); - } - } - } - return otherMap; - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/interactor/properties/StdPropsLoaderTest.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/interactor/properties/StdPropsLoaderTest.java deleted file mode 100644 index 9bef89b3f3..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/interactor/properties/StdPropsLoaderTest.java +++ /dev/null @@ -1,119 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.interactor.properties; - -import org.apache.linkis.cli.common.entity.properties.ClientProperties; -import org.apache.linkis.cli.core.interactor.properties.reader.PropertiesReader; - -import java.util.Arrays; -import java.util.List; -import java.util.stream.Collectors; - -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -public class StdPropsLoaderTest { - PropertiesLoader loader; - - @BeforeEach - public void before() throws Exception { - System.setProperty("conf.root", "src/test/resources/conf/"); - System.setProperty("conf.file", "linkis-cli.properties"); - String configPath = System.getProperty("conf.root"); - String defaultConfFileName = System.getProperty("conf.file"); - /* - default config, -Dconf.root & -Dconf.file specifies config path - */ - List readersList = - new PropsFilesScanner().getPropsReaders(configPath); // +1 user config - /* - user defined config - */ - // load all config files - loader = - new StdPropsLoader() - .addPropertiesReaders(readersList.toArray(new PropertiesReader[readersList.size()])); - } - - @AfterEach - public void after() throws Exception {} - - /** Method: setPropertiesReaders(PropertiesReader[] readers) */ - @Test - public void testSetPropertiesReaders() throws Exception { - // TODO: Test goes here... - loader.setPropertiesReaders(new PropertiesReader[0]); - } - - /** Method: getAllReadersAsMap() */ - @Test - public void testGetAllReadersAsMap() throws Exception { - // TODO: Test goes here... - } - - /** Method: addPropertiesReader(PropertiesReader reader) */ - @Test - public void testAddPropertiesReader() throws Exception { - // TODO: Test goes here... - } - - /** Method: addPropertiesReaders(PropertiesReader[] readers) */ - @Test - public void testAddPropertiesReaders() throws Exception { - // TODO: Test goes here... - } - - /** Method: getPropertiesReader(String identifier) */ - @Test - public void testGetPropertiesReader() throws Exception { - // TODO: Test goes here... - } - - /** Method: removePropertiesReader(String identifier) */ - @Test - public void testRemovePropertiesReader() throws Exception { - // TODO: Test goes here... - } - - /** Method: loadProperties() */ - @Test - public void testLoadProperties() throws Exception { - ClientProperties[] loaderResult = loader.loadProperties(); - - List properties = - Arrays.stream(loaderResult) - .sorted((p1, p2) -> p1.size() - p2.size()) - .collect(Collectors.toList()); - - assertEquals(2, properties.size()); - assertEquals(properties.get(0).getPropsId(), "user.properties"); - assertEquals(properties.get(0).size(), 4); - - assertEquals(properties.get(1).getPropsId(), "linkis-cli.properties"); - assertEquals(properties.get(1).size(), 8); - } - - /** Method: checkInit() */ - @Test - public void testCheckInit() throws Exception { - // TODO: Test goes here... - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/interactor/var/StdVarAccessTest.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/interactor/var/StdVarAccessTest.java deleted file mode 100644 index 497a539dd5..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/interactor/var/StdVarAccessTest.java +++ /dev/null @@ -1,298 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.interactor.var; - -import org.apache.linkis.cli.common.entity.command.CmdTemplate; -import org.apache.linkis.cli.common.entity.properties.ClientProperties; -import org.apache.linkis.cli.common.entity.var.VarAccess; -import org.apache.linkis.cli.core.constants.TestConstants; -import org.apache.linkis.cli.core.interactor.command.TestCmdType; -import org.apache.linkis.cli.core.interactor.command.fitter.SingleTplFitter; -import org.apache.linkis.cli.core.interactor.command.parser.Parser; -import org.apache.linkis.cli.core.interactor.command.parser.SingleCmdParser; -import org.apache.linkis.cli.core.interactor.command.parser.result.ParseResult; -import org.apache.linkis.cli.core.interactor.command.template.TestSparkCmdTemplate; -import org.apache.linkis.cli.core.interactor.properties.PropertiesLoader; -import org.apache.linkis.cli.core.interactor.properties.PropsFilesScanner; -import org.apache.linkis.cli.core.interactor.properties.StdPropsLoader; -import org.apache.linkis.cli.core.interactor.properties.reader.PropertiesReader; -import org.apache.linkis.cli.core.interactor.properties.reader.PropsFileReader; -import org.apache.linkis.cli.core.interactor.validate.ParsedTplValidator; - -import org.apache.commons.lang3.StringUtils; - -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; - -public class StdVarAccessTest { - String[] cmdStr; - VarAccess stdVarAccess; - - @BeforeEach - public void before() throws Exception { - cmdStr = - new String[] { - "-u", - "hadoop", - "-pwd", - "1234", - "-c", - "src/test/resources/conf/user.properties", - "--cmd", - "show tables", - "--split", - "\',\'", - "--queue", - "q05", - "--name", - "testApp", - // "--hiveconf", "/path/...", - "--num-executors", - "4", - "--executor-cores", - "4", - "--executor-memory", - "4G", - "--shuffle-partitions", - "200", - "--other", - "--other-spark-config=none", - }; - - TestSparkCmdTemplate template = new TestSparkCmdTemplate(); - Map templateMap = new HashMap<>(); - templateMap.put(template.getCmdType().getName(), template); - - Parser parser = - new SingleCmdParser() - .setMapper(null) - .setTemplate(templateMap.get(TestCmdType.SPARK.getName())) - .setFitter(new SingleTplFitter()); - - ParseResult result = parser.parse(cmdStr); - - ParsedTplValidator parsedTplValidator = new ParsedTplValidator(); - parsedTplValidator.doValidation(result.getParsedTemplate()); - - System.setProperty("conf.root", "src/test/resources/conf/"); - System.setProperty("conf.file", "linkis-cli.properties"); - String configPath = System.getProperty("conf.root"); - String defaultConfFileName = System.getProperty("conf.file"); - /* - default config, -Dconf.root & -Dconf.file specifies config path - */ - List readersList = - new PropsFilesScanner().getPropsReaders(configPath); // +1 user config - /* - user defined config - */ - String userConfPath = - (String) result.getParams().getParamItemMap().get(TestConstants.PARAM_USR_CONF).getValue(); - if (StringUtils.isNotBlank(userConfPath)) { - PropertiesReader reader = new PropsFileReader(); - reader.setPropsId("user.conf"); - reader.setPropsPath(userConfPath); - readersList.add(reader); - } else { - } - // load all config files - PropertiesLoader loader = - new StdPropsLoader() - .addPropertiesReaders(readersList.toArray(new PropertiesReader[readersList.size()])); - ClientProperties[] loaderResult = loader.loadProperties(); - Map propertiesMap = new HashMap<>(); - for (ClientProperties properties : loaderResult) { - propertiesMap.put(properties.getPropsId(), properties); - } - - stdVarAccess = - new StdVarAccess() - .setCmdParams(result.getParams()) - .setUserConf(propertiesMap.get("user.conf")) - .setDefaultConf(propertiesMap.get(defaultConfFileName)) - .init(); - } - - @AfterEach - public void after() throws Exception {} - - /** Method: setPrimaryParam(Params primaryParam) */ - @Test - public void testSetPrimaryParam() throws Exception { - // TODO: Test goes here... - } - - /** Method: getPrimaryParam(String identifier) */ - @Test - public void testGetPrimaryParam() throws Exception { - // TODO: Test goes here... - } - - /** Method: setCmdParams(Params subParam) */ - @Test - public void testSetSubParam() throws Exception { - // TODO: Test goes here... - } - - /** Method: getSubParam(String identifier) */ - @Test - public void testGetSubParam() throws Exception { - // TODO: Test goes here... - } - - /** Method: setUserConf(ClientProperties userConf) */ - @Test - public void testSetUserConf() throws Exception { - // TODO: Test goes here... - } - - /** Method: getUserConf(String identifier) */ - @Test - public void testGetUserConf() throws Exception { - // TODO: Test goes here... - } - - /** Method: setDefaultConf(ClientProperties defaultConf) */ - @Test - public void testSetDefaultConf() throws Exception { - // TODO: Test goes here... - } - - /** Method: getDefaultConf(String identifier) */ - @Test - public void testGetDefaultConf() throws Exception { - // TODO: Test goes here... - } - - /** Method: checkInit() */ - @Test - public void testCheckInit() throws Exception { - // TODO: Test goes here... - } - - /** Method: getVar(Class clazz, String key) */ - @Test - public void testGetVar() throws Exception { - System.out.println(stdVarAccess.getVar(String.class, TestConstants.PARAM_COMMON_CMD)); - assertEquals(stdVarAccess.getVar(String.class, TestConstants.PARAM_COMMON_CMD), "show tables"); - System.out.println(stdVarAccess.getVar(String.class, "user.props")); - assertEquals(stdVarAccess.getVar(String.class, "wds.linkis.client.not.exist"), null); - System.out.println( - stdVarAccess.getVar( - Integer.class, TestConstants.PARAM_SPARK_EXECUTOR_CORES)); // see if priority works - assertEquals( - (long) stdVarAccess.getVar(Integer.class, TestConstants.PARAM_SPARK_EXECUTOR_CORES), 4); - assertEquals((long) stdVarAccess.getVar(Integer.class, "conf.prop.integer"), 9); - assertEquals(stdVarAccess.getVar(String.class, "conf.prop.string"), "str"); - assertEquals( - stdVarAccess.getVar(String.class, "wds.linkis.client.param.conf.spark.executor.memory"), - "11111G"); - - System.out.println(stdVarAccess.getAllVarKeys().length); - System.out.println(Arrays.toString(stdVarAccess.getAllVarKeys())); - assertTrue(stdVarAccess.getAllVarKeys().length != 0); - } - - /** Method: getVarOrDefault(Class clazz, String key, T defaultValue) */ - @Test - public void testGetVarOrDefault() throws Exception { - // TODO: Test goes here... - } - - /** Method: getAllVarKeys() */ - @Test - public void testGetAllVarKeys() throws Exception { - // TODO: Test goes here... - } - - /** Method: getVarFromParam(Class clazz, String key, Params param) */ - @Test - public void testGetVarFromParam() throws Exception { - // TODO: Test goes here... - /* - try { - Method method = StdVarAccess.getClass().getMethod("getVarFromParam", Class.class, String.class, Params.class); - method.setAccessible(true); - method.invoke(, ); - } catch(NoSuchMethodException e) { - } catch(IllegalAccessException e) { - } catch(InvocationTargetException e) { - } - */ - } - - /** Method: getDefaultVarFromParam(Class clazz, String key, Params param) */ - @Test - public void testGetDefaultVarFromParam() throws Exception { - // TODO: Test goes here... - /* - try { - Method method = StdVarAccess.getClass().getMethod("getDefaultVarFromParam", Class.class, String.class, Params.class); - method.setAccessible(true); - method.invoke(, ); - } catch(NoSuchMethodException e) { - } catch(IllegalAccessException e) { - } catch(InvocationTargetException e) { - } - */ - } - - /** Method: getVarFromCfg(Class clazz, String key, ClientProperties conf) */ - @Test - public void testGetVarFromCfg() throws Exception { - // TODO: Test goes here... - /* - try { - Method method = StdVarAccess.getClass().getMethod("getVarFromCfg", Class.class, String.class, ClientProperties.class); - method.setAccessible(true); - method.invoke(, ); - } catch(NoSuchMethodException e) { - } catch(IllegalAccessException e) { - } catch(InvocationTargetException e) { - } - */ - } - - /** - * Method: getVarFromCfgGivenConverter(String key, ClientProperties conf, - * AbstractStringConverter converter) - */ - @Test - public void testGetVarFromCfgGivenConverter() throws Exception { - // TODO: Test goes here... - /* - try { - Method method = StdVarAccess.getClass().getMethod("getVarFromCfgGivenConverter", String.class, ClientProperties.class, AbstractStringConverter.class); - method.setAccessible(true); - method.invoke(, ); - } catch(NoSuchMethodException e) { - } catch(IllegalAccessException e) { - } catch(InvocationTargetException e) { - } - */ - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/utils/TestUtils.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/utils/TestUtils.java deleted file mode 100644 index 61cdd6aa7f..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/utils/TestUtils.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.utils; - -import org.apache.commons.io.FileUtils; -import org.apache.commons.lang3.StringUtils; - -import java.io.File; -import java.util.HashMap; -import java.util.Map; - -public class TestUtils { - private static String replaceParas(String context, Map m) { - if (context == null) { - return null; - } - for (Map.Entry entry : m.entrySet()) { - if (entry.getKey() == null || entry.getValue() == null) { - continue; - } - String key = "[#" + entry.getKey().trim() + "]"; - String value = entry.getValue().trim(); - - context = StringUtils.replace(context, key, value); - } - return context; - } - - public static Map parseArgMap(String str, String separator) { - - Map argsProps = new HashMap<>(); - String[] args = StringUtils.splitByWholeSeparator(str, separator); - - for (String arg : args) { - int index = arg.indexOf("="); - if (index != -1) { - argsProps.put(arg.substring(0, index).trim(), arg.substring(index + 1).trim()); - } - } - return argsProps; - } - - public static String readShellFileAndReplaceParas(String filename, String argsStr, String split) - throws Exception { - - String fileContent; - - File inputFile = new File(filename); - - fileContent = FileUtils.readFileToString(inputFile); - - Map argsMap = parseArgMap(argsStr, split); - - fileContent = replaceParas(fileContent, argsMap); - - return fileContent; - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/resources/conf/linkis-cli.properties b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/resources/conf/linkis-cli.properties deleted file mode 100644 index e0f85cf3e5..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/resources/conf/linkis-cli.properties +++ /dev/null @@ -1,34 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# -wds.linkis.client.common.creator=IDE3 -wds.linkis.client.common.gatewayUrl=http://127.0.0.1:8088 -#wds.linkis.client.common.authStrategy -#wds.linki.clients.common.tokenKey -#wds.linkis.client.common.tokenValue -# -#wds.linkis.client.common.submitUser -#wds.linkis.client.common.submitPassword -#wds.linkis.client.common.proxyUser -wds.linkis.client.param.conf.wds.linkis.yarnqueue=q02 -wds.linkis.client.param.conf.yarnqueue.cores.max=233 -wds.linkis.client.param.conf.yarnqueue.memory.max=233G -wds.linkis.client.param.conf.spark.executor.instances=9 -wds.linkis.client.param.conf.spark.executor.cores=9 -wds.linkis.client.param.conf.spark.executor.memory=9 -#wds.linkis.client.label="key1=val1,key2=val2" -#wds.linkis.client.param.conf="key1=val1,key2=val2" -#wds.linkis.client.param.var="key1=val1,key2=val2" -#wds.linkis.client.source="key1=val1,key2=val2" \ No newline at end of file diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/resources/log4j2.xml b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/resources/log4j2.xml deleted file mode 100644 index c744678876..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/resources/log4j2.xml +++ /dev/null @@ -1,50 +0,0 @@ - - - - - - ${sys:log.path} - ${sys:log.file} - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/linkis-computation-governance/linkis-client/linkis-cli/pom.xml b/linkis-computation-governance/linkis-client/linkis-cli/pom.xml index e9efc432d5..76723c4adb 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/pom.xml +++ b/linkis-computation-governance/linkis-client/linkis-cli/pom.xml @@ -25,15 +25,24 @@ ../../../pom.xml linkis-cli - pom - - - linkis-cli-common - linkis-cli-core - linkis-cli-application - + jar + + org.apache.linkis + linkis-computation-client + ${project.version} + + + org.apache.linkis + linkis-gateway-httpclient-support + ${project.version} + + + org.reflections + reflections + ${reflections.version} + org.apache.linkis linkis-common @@ -42,17 +51,53 @@ org.apache.commons commons-lang3 + ${commons-lang3.version} com.google.code.gson gson ${gson.version} - ${project.artifactId}-${project.version} + + + true + ${basedir}/src/main/resources + + + + + org.apache.maven.plugins + maven-assembly-plugin + false + + false + out + false + false + + /src/main/assembly/distribution.xml + + + + + make-assembly + + single + + package + + + /src/main/assembly/distribution.xml + + + + + + diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/assembly/distribution.xml b/linkis-computation-governance/linkis-client/linkis-cli/src/main/assembly/distribution.xml similarity index 98% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/assembly/distribution.xml rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/assembly/distribution.xml index a1cf04696a..24463e39af 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/assembly/distribution.xml +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/assembly/distribution.xml @@ -16,7 +16,7 @@ ~ limitations under the License. --> - linkis-cli diff --git a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/CtxBuilder.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/CtxBuilder.java new file mode 100644 index 0000000000..cfa57d4e26 --- /dev/null +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/CtxBuilder.java @@ -0,0 +1,185 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.cli.application; + +import org.apache.linkis.cli.application.constants.CliConstants; +import org.apache.linkis.cli.application.constants.CliKeys; +import org.apache.linkis.cli.application.entity.command.CmdTemplate; +import org.apache.linkis.cli.application.entity.command.Params; +import org.apache.linkis.cli.application.entity.context.CliCtx; +import org.apache.linkis.cli.application.entity.var.VarAccess; +import org.apache.linkis.cli.application.exception.LinkisClientRuntimeException; +import org.apache.linkis.cli.application.exception.PropsException; +import org.apache.linkis.cli.application.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.exception.error.ErrorLevel; +import org.apache.linkis.cli.application.interactor.command.CliCmdType; +import org.apache.linkis.cli.application.interactor.command.CmdTemplateFactory; +import org.apache.linkis.cli.application.interactor.command.fitter.SingleTplFitter; +import org.apache.linkis.cli.application.interactor.command.parser.Parser; +import org.apache.linkis.cli.application.interactor.command.parser.SingleCmdParser; +import org.apache.linkis.cli.application.interactor.command.parser.result.ParseResult; +import org.apache.linkis.cli.application.interactor.context.CliCtxImpl; +import org.apache.linkis.cli.application.interactor.properties.ClientProperties; +import org.apache.linkis.cli.application.interactor.properties.PropertiesLoader; +import org.apache.linkis.cli.application.interactor.properties.PropsFilesScanner; +import org.apache.linkis.cli.application.interactor.properties.reader.PropertiesReader; +import org.apache.linkis.cli.application.interactor.properties.reader.PropsFileReader; +import org.apache.linkis.cli.application.interactor.validate.ParsedTplValidator; +import org.apache.linkis.cli.application.interactor.var.VarAccessImpl; +import org.apache.linkis.cli.application.utils.CliUtils; +import org.apache.linkis.cli.application.utils.LoggerManager; + +import org.apache.commons.lang3.StringUtils; + +import java.io.*; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class CtxBuilder { + private static Logger logger = LoggerFactory.getLogger(CtxBuilder.class); + + /** generate Templates load env variables TODO: load version info */ + public static CliCtx buildCtx(String[] args) throws LinkisClientRuntimeException { + /* + user input + */ + CmdTemplate template = CmdTemplateFactory.getTemplateCopy(CliCmdType.UNIVERSAL); + Parser parser = + new SingleCmdParser() + .setMapper(null) + .setTemplate(template) + .setFitter(new SingleTplFitter()); + + ParseResult result = parser.parse(args); + + ParsedTplValidator parsedTplValidator = new ParsedTplValidator(); + + parsedTplValidator.doValidation(result.getParsedTemplate()); + + Params params = result.getParams(); + logger.debug("==========params============\n" + CliUtils.GSON.toJson(params)); + + /* + VarAccess for sys_prop, sys_env + */ + + Map propertiesMap = new HashMap<>(); + + LoggerManager.getInformationLogger() + .info( + "LogFile path: " + + System.getProperty(CliKeys.LOG_PATH_KEY) + + "/" + + System.getProperty(CliKeys.LOG_FILE_KEY)); + /* + default config, -Dconf.root & -Dconf.file specifies config path + */ + // scan config files given root path + String configPath = System.getProperty(CliKeys.CLIENT_CONFIG_ROOT_KEY); + String defaultConfFileName = + System.getProperty(CliKeys.DEFAULT_CONFIG_FILE_NAME_KEY, CliConstants.DEFAULT_CONFIG_NAME); + + if (StringUtils.isBlank(configPath)) { + throw new PropsException( + "PRP0007", + ErrorLevel.ERROR, + CommonErrMsg.PropsLoaderErr, + "configuration root path specified by env variable: " + + CliKeys.CLIENT_CONFIG_ROOT_KEY + + " is empty."); + } + + List readersList = + new PropsFilesScanner().getPropsReaders(configPath); // +1 user config + /* + user defined config + */ + String userConfPath = null; + if (params.containsParam(CliKeys.LINKIS_CLIENT_USER_CONFIG)) { + userConfPath = + (String) params.getParamItemMap().get(CliKeys.LINKIS_CLIENT_USER_CONFIG).getValue(); + } + if (StringUtils.isNotBlank(userConfPath)) { + PropertiesReader reader = + new PropsFileReader() + .setPropsId(CliKeys.LINKIS_CLIENT_USER_CONFIG) + .setPropsPath(userConfPath); + readersList.add(reader); + } else { + LoggerManager.getInformationLogger() + .info("User does not provide usr-configuration file. Will use default config"); + } + /* + load properties + */ + PropertiesLoader loader = + new PropertiesLoader() + .addPropertiesReaders(readersList.toArray(new PropertiesReader[readersList.size()])); + ClientProperties[] loaderResult = loader.loadProperties(); + for (ClientProperties properties : loaderResult) { + if (StringUtils.equals(properties.getPropsId(), CliKeys.LINKIS_CLIENT_USER_CONFIG)) { + for (Map.Entry prop : properties.entrySet()) { + if (StringUtils.startsWith( + (String) prop.getKey(), CliKeys.LINKIS_CLIENT_NONCUSTOMIZABLE)) { + throw new PropsException( + "PRP0007", + ErrorLevel.ERROR, + CommonErrMsg.PropsLoaderErr, + "User cannot specify non-customizable configuration: " + prop.getKey()); + } + } + } + propertiesMap.put(properties.getPropsId(), properties); + } + + /* + VarAccess for cmd, config + */ + VarAccess varAccess = + new VarAccessImpl() + .setCmdParams(params) + .setUserConf(propertiesMap.get(CliKeys.LINKIS_CLIENT_USER_CONFIG)) + .setDefaultConf(propertiesMap.get(defaultConfFileName)) + .init(); + logger.info("==========std_var============\n" + CliUtils.GSON.toJson(varAccess)); + + Properties props = new Properties(); + try (InputStream inputStream = + CtxBuilder.class.getClassLoader().getResourceAsStream("version.properties")) { + try (InputStreamReader reader = new InputStreamReader(inputStream)) { + try (BufferedReader bufferedReader = new BufferedReader(reader)) { + props.load(bufferedReader); + } + } + } catch (Exception e) { + logger.warn("Failed to load version info", e); + } + + String verion = props.getProperty(CliKeys.VERSION); + + Map extraMap = new HashMap<>(); + extraMap.put(CliKeys.VERSION, verion); + + return new CliCtxImpl(params.getCmdType(), template, varAccess, extraMap); + } +} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/LinkisClientApplication.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/LinkisClientApplication.java new file mode 100644 index 0000000000..1fb21043a1 --- /dev/null +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/LinkisClientApplication.java @@ -0,0 +1,193 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.cli.application; + +import org.apache.linkis.cli.application.constants.CliConstants; +import org.apache.linkis.cli.application.constants.CliKeys; +import org.apache.linkis.cli.application.entity.command.CmdTemplate; +import org.apache.linkis.cli.application.entity.context.CliCtx; +import org.apache.linkis.cli.application.entity.job.Job; +import org.apache.linkis.cli.application.entity.job.JobResult; +import org.apache.linkis.cli.application.exception.CommandException; +import org.apache.linkis.cli.application.interactor.command.CmdTemplateFactory; +import org.apache.linkis.cli.application.interactor.command.template.UniversalCmdTemplate; +import org.apache.linkis.cli.application.interactor.job.help.HelpJob; +import org.apache.linkis.cli.application.interactor.job.interactive.InteractiveJob; +import org.apache.linkis.cli.application.interactor.job.jobcmd.JobCmdJob; +import org.apache.linkis.cli.application.interactor.job.once.LinkisOnceJob; +import org.apache.linkis.cli.application.interactor.job.version.VersionJob; +import org.apache.linkis.cli.application.operator.OperManager; +import org.apache.linkis.cli.application.operator.once.OnceOperBuilder; +import org.apache.linkis.cli.application.operator.ujes.LinkisOperBuilder; +import org.apache.linkis.cli.application.present.HelpPresenter; +import org.apache.linkis.cli.application.present.model.HelpInfoModel; +import org.apache.linkis.cli.application.utils.LoggerManager; +import org.apache.linkis.cli.application.utils.SchedulerManager; + +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.exception.ExceptionUtils; + +import java.util.HashMap; +import java.util.Map; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class LinkisClientApplication { + private static Logger logger = LoggerFactory.getLogger(LinkisClientApplication.class); + + private static boolean showHelp = false; + + public static void main(String[] args) { + /* + generate template + */ + CmdTemplateFactory.register(new UniversalCmdTemplate()); + + /* + build ctx + */ + CliCtx ctx = null; + try { + ctx = CtxBuilder.buildCtx(args); + } catch (CommandException e) { + CmdTemplate template = CmdTemplateFactory.getTemplateOri(e.getCmdType()); + if (template != null) { + HelpInfoModel model = new HelpInfoModel(); + model.buildModel(ctx.getTemplate()); + new HelpPresenter().present(model); + } + LoggerManager.getInformationLogger().error("Failed to build CliCtx", e); + System.exit(-1); + } + + /* + prepare oper + */ + OperManager.register(CliKeys.Linkis_OPER, new LinkisOperBuilder()); + OperManager.register(CliKeys.LINKIS_ONCE, new OnceOperBuilder()); + + /* + run job + */ + Job job; + if (isVersionCmd(ctx)) { + job = new VersionJob(); + } else if (isHelp(ctx)) { + job = new HelpJob(); + } else if (isJobCmd(ctx)) { + job = new JobCmdJob(); + } else if (isOnceCmd(ctx)) { + job = new LinkisOnceJob(); + } else { + job = new InteractiveJob(); + } + job.build(ctx); + JobResult result; + try { + Runtime.getRuntime() + .addShutdownHook( + new Thread( + () -> { + if (job != null) { + job.onDestroy(); + } + })); + result = job.run(); + } catch (Exception e) { + logger.error("Failed to execute job", e); + result = + new JobResult() { + @Override + public Boolean isSuccess() { + return false; + } + + @Override + public String getMessage() { + return "Failed to execute job" + ExceptionUtils.getStackTrace(e); + } + + @Override + public Map getExtraMessage() { + return new HashMap<>(); + } + }; + } + + /* + process result + */ + printIndicator(result); + + SchedulerManager.shutDown(); + + if (result.isSuccess()) { + System.exit(0); + } else { + System.exit(-1); + } + } + + private static void printIndicator(JobResult jobResult) { + if (jobResult.isSuccess()) { + LoggerManager.getPlaintTextLogger().info(CliConstants.SUCCESS_INDICATOR); + } else { + LoggerManager.getPlaintTextLogger().info(jobResult.getMessage()); + StringBuilder b = new StringBuilder(); + for (Map.Entry e : jobResult.getExtraMessage().entrySet()) { + b.append(e.getKey()).append(":").append(e.getValue()).append(System.lineSeparator()); + } + LoggerManager.getPlaintTextLogger().info(b.toString()); + LoggerManager.getPlaintTextLogger().info(CliConstants.FAILURE_INDICATOR); + } + } + + private static Boolean isHelp(CliCtx ctx) { + if (ctx.getVarAccess().hasVar(CliKeys.LINKIS_CLIENT_HELP_OPT)) { + return true; + } + return false; + } + + private static Boolean isVersionCmd(CliCtx ctx) { + if (ctx.getVarAccess().hasVar(CliKeys.VERSION)) { + return true; + } + return false; + } + + private static Boolean isJobCmd(CliCtx ctx) { + if (ctx.getVarAccess().hasVar(CliKeys.LINKIS_CLIENT_KILL_OPT) + || ctx.getVarAccess().hasVar(CliKeys.LINKIS_CLIENT_STATUS_OPT) + || ctx.getVarAccess().hasVar(CliKeys.LINKIS_CLIENT_DESC_OPT) + || ctx.getVarAccess().hasVar(CliKeys.LINKIS_CLIENT_LOG_OPT) + || ctx.getVarAccess().hasVar(CliKeys.LINKIS_CLIENT_RESULT_OPT) + || ctx.getVarAccess().hasVar(CliKeys.LINKIS_CLIENT_LIST_OPT)) { + return true; + } + return false; + } + + private static Boolean isOnceCmd(CliCtx ctx) { + String mode = + ctx.getVarAccess() + .getVarOrDefault(String.class, CliKeys.LINKIS_CLIENT_MODE_OPT, CliConstants.UJES_MODE); + return StringUtils.equalsIgnoreCase(mode, CliConstants.ONCE_MODE); + } +} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/constants/AppConstants.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/constants/CliConstants.java similarity index 66% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/constants/AppConstants.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/constants/CliConstants.java index 711fc2d156..d277f435fe 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/constants/AppConstants.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/constants/CliConstants.java @@ -17,7 +17,7 @@ package org.apache.linkis.cli.application.constants; -public class AppConstants { +public class CliConstants { public static final String DEFAULT_CONFIG_NAME = "linkis-cli.properties"; public static final Long JOB_QUERY_SLEEP_MILLS = 2000l; @@ -41,4 +41,29 @@ public class AppConstants { public static final String UJES_MODE = "ujes"; public static final String ONCE_MODE = "once"; + + public static final Integer REQUEST_MAX_RETRY_TIME = 3; + + public static final String UNIVERSAL_SUBCMD = "linkis-cli"; + + public static final String UNIVERSAL_SUBCMD_DESC = + "command for all types of jobs supported by Linkis"; + + public static final String SUCCESS_INDICATOR = "############Execute Success!!!########"; + + public static final String FAILURE_INDICATOR = "############Execute Error!!!########"; + + public static final String ARRAY_SEQ = "@#@"; + + public static final String ARRAY_SEQ_REGEX = "(?=([^\"]*\"[^\"]*\")*[^\"]*$)"; + + public static final int MAX_NUM_OF_COMMAND_ARGUEMENTS = 10; + + public static final String CONFIG_DIR = "config.path"; + + public static final String[] CONFIG_EXTENSION = {"properties"}; + + public static final String SYSTEM_PROPERTIES_IDENTIFIER = "SYS_PROP"; + + public static final String SYSTEM_ENV_IDENTIFIER = "SYS_ENV"; } diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/constants/AppKeys.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/constants/CliKeys.java similarity index 95% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/constants/AppKeys.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/constants/CliKeys.java index 7a38d9191f..966836e0bf 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/constants/AppKeys.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/constants/CliKeys.java @@ -17,7 +17,7 @@ package org.apache.linkis.cli.application.constants; -public class AppKeys { +public class CliKeys { /** User Not configurable */ public static final String ADMIN_USERS = "hadoop,root,shangda"; @@ -38,6 +38,8 @@ public class AppKeys { public static final String DEFAULT_CONFIG_FILE_NAME_KEY = "conf.file"; public static final String LINUX_USER_KEY = "user.name"; + public static final String VERSION = "cli.version"; + /** Configurable */ /* execution type @@ -75,6 +77,8 @@ public class AppKeys { JOB_LABEL + "." + LinkisKeys.KEY_CODETYPE; // corresponds to server api. public static final String JOB_LABEL_EXECUTEONCE = JOB_LABEL + "." + LinkisKeys.KEY_EXECUTEONCE; // corresponds to server api. + public static final String JOB_LABEL_CLUSTER = + JOB_LABEL + "." + LinkisKeys.KEY_CLUSTER; // corresponds to server api. /* Job command @@ -139,6 +143,6 @@ public class AppKeys { public static final String LINKIS_CLIENT_COMMON_RESULT_SET_PAGE_SIZE = LINKIS_CLIENT_COMMON + ".resultset.page.size"; - public static final String REUSABLE_UJES_CLIENT = "reusable.ujes.client"; - public static final String REUSABLE_ONCEJOB_CLIENT = "reusable.onceJob.client"; + public static final String Linkis_OPER = "linkis.oper"; + public static final String LINKIS_ONCE = "linkis.once"; } diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/constants/LinkisConstants.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/constants/LinkisConstants.java similarity index 100% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/constants/LinkisConstants.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/constants/LinkisConstants.java diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/constants/LinkisKeys.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/constants/LinkisKeys.java similarity index 95% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/constants/LinkisKeys.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/constants/LinkisKeys.java index 9488c25f4a..170f1a8f86 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/constants/LinkisKeys.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/constants/LinkisKeys.java @@ -34,6 +34,7 @@ public class LinkisKeys { public static final String KEY_EXECID = "execId"; public static final String KEY_UMUSER = "umUser"; public static final String KEY_EXECUTEONCE = "executeOnce"; + public static final String KEY_CLUSTER = "yarnCluster"; public static final String KEY_TENANT = "tenant"; public static final String META_DATA_COLUMN_NAME = "columnName"; public static final String KEY_SHELL_WORKING_DIRECTORY = @@ -46,4 +47,5 @@ public class LinkisKeys { public static final String KEY_YARN_QUEUE = "wds.linkis.rm.yarnqueue"; public static final String KEY_HIVE_RESULT_DISPLAY_TBALE = "hive.resultset.use.unique.column.names"; + public static final String CLI_VERSION = "cli.version"; } diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/command/CmdOption.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/entity/command/CmdOption.java similarity index 95% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/command/CmdOption.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/entity/command/CmdOption.java index f9b6924be5..1583090c88 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/command/CmdOption.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/entity/command/CmdOption.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.linkis.cli.common.entity.command; +package org.apache.linkis.cli.application.entity.command; public interface CmdOption extends Cloneable { String getParamName(); diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/command/CmdTemplate.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/entity/command/CmdTemplate.java similarity index 94% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/command/CmdTemplate.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/entity/command/CmdTemplate.java index da802bc5ff..94677774a8 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/command/CmdTemplate.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/entity/command/CmdTemplate.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.linkis.cli.common.entity.command; +package org.apache.linkis.cli.application.entity.command; import java.util.List; import java.util.Map; diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/command/CmdType.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/entity/command/CmdType.java similarity index 93% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/command/CmdType.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/entity/command/CmdType.java index 6819fdb92b..2f9dd803e8 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/command/CmdType.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/entity/command/CmdType.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.linkis.cli.common.entity.command; +package org.apache.linkis.cli.application.entity.command; public interface CmdType extends Cloneable { diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/command/ParamItem.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/entity/command/ParamItem.java similarity index 96% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/command/ParamItem.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/entity/command/ParamItem.java index 02a33cfc95..c8a28d9009 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/command/ParamItem.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/entity/command/ParamItem.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.linkis.cli.common.entity.command; +package org.apache.linkis.cli.application.entity.command; public class ParamItem { private String keyPrefix; diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/command/Params.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/entity/command/Params.java similarity index 97% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/command/Params.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/entity/command/Params.java index 9fa73c01fc..558521ad72 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/command/Params.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/entity/command/Params.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.linkis.cli.common.entity.command; +package org.apache.linkis.cli.application.entity.command; import java.util.Map; diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/execution/Execution.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/entity/context/CliCtx.java similarity index 62% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/execution/Execution.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/entity/context/CliCtx.java index 23e99601eb..f18a402c2e 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/execution/Execution.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/entity/context/CliCtx.java @@ -15,21 +15,25 @@ * limitations under the License. */ -package org.apache.linkis.cli.common.entity.execution; +package org.apache.linkis.cli.application.entity.context; -import org.apache.linkis.cli.common.entity.job.Job; -import org.apache.linkis.cli.common.entity.result.ExecutionResult; +import org.apache.linkis.cli.application.entity.command.CmdTemplate; +import org.apache.linkis.cli.application.entity.command.CmdType; +import org.apache.linkis.cli.application.entity.var.VarAccess; import java.util.Map; -public interface Execution { - /* - start execution - */ - ExecutionResult execute(Map jobs); +public interface CliCtx { + /** + * accessing and passing VarAccess instance between jobs + * + * @return + */ + CmdType getCmdType(); - /* - terminate execution(often used in shutdown hook) - */ - boolean terminate(Map jobs); + CmdTemplate getTemplate(); + + VarAccess getVarAccess(); + + Map getExtraMap(); } diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/present/display/DisplayOperator.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/entity/job/Job.java similarity index 79% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/present/display/DisplayOperator.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/entity/job/Job.java index 228b75856d..303c530c2a 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/present/display/DisplayOperator.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/entity/job/Job.java @@ -15,10 +15,14 @@ * limitations under the License. */ -package org.apache.linkis.cli.core.present.display; +package org.apache.linkis.cli.application.entity.job; -import org.apache.linkis.cli.core.present.display.data.DisplayData; +import org.apache.linkis.cli.application.entity.context.CliCtx; -public interface DisplayOperator { - void doOutput(DisplayData data); +public interface Job { + void build(CliCtx ctx); + + JobResult run(); + + void onDestroy(); } diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/LogCallbackService.scala b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/entity/job/JobResult.java similarity index 80% rename from linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/LogCallbackService.scala rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/entity/job/JobResult.java index 873b098194..eba7b7ba8a 100644 --- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/LogCallbackService.scala +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/entity/job/JobResult.java @@ -15,12 +15,14 @@ * limitations under the License. */ -package org.apache.linkis.ecm.server.service +package org.apache.linkis.cli.application.entity.job; -import org.apache.linkis.protocol.callback.LogCallbackProtocol +import java.util.Map; -trait LogCallbackService { +public interface JobResult { + Boolean isSuccess(); - def dealLog(protocol: LogCallbackProtocol): Unit + String getMessage(); + Map getExtraMessage(); } diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/job/JobStatus.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/entity/job/JobStatus.java similarity index 95% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/job/JobStatus.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/entity/job/JobStatus.java index d0b6f445b6..7f048dd88a 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/job/JobStatus.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/entity/job/JobStatus.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.linkis.cli.common.entity.job; +package org.apache.linkis.cli.application.entity.job; public interface JobStatus { diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/version/Version.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/entity/operator/JobOper.java similarity index 90% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/version/Version.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/entity/operator/JobOper.java index 7f005b0bbd..ae1bf7c465 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/version/Version.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/entity/operator/JobOper.java @@ -15,6 +15,6 @@ * limitations under the License. */ -package org.apache.linkis.cli.common.entity.version; +package org.apache.linkis.cli.application.entity.operator; -public interface Version {} +public interface JobOper {} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/present/Model.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/entity/present/Model.java similarity index 93% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/present/Model.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/entity/present/Model.java index b68fd986aa..9fe2c106fc 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/present/Model.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/entity/present/Model.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.linkis.cli.common.entity.present; +package org.apache.linkis.cli.application.entity.present; public interface Model { void buildModel(Object data); diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/present/Presenter.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/entity/present/Presenter.java similarity index 88% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/present/Presenter.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/entity/present/Presenter.java index dbe1adaefe..7c27316dfb 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/present/Presenter.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/entity/present/Presenter.java @@ -15,8 +15,8 @@ * limitations under the License. */ -package org.apache.linkis.cli.common.entity.present; +package org.apache.linkis.cli.application.entity.present; public interface Presenter { - void present(Model model, PresentWay presentWay); + void present(Model model); } diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/var/VarAccess.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/entity/var/VarAccess.java similarity index 94% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/var/VarAccess.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/entity/var/VarAccess.java index 50df2236e1..8236bd81c9 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/var/VarAccess.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/entity/var/VarAccess.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.linkis.cli.common.entity.var; +package org.apache.linkis.cli.application.entity.var; /** * @program: linkis-cli @@ -24,8 +24,6 @@ */ public interface VarAccess { - void checkInit(); - T getVar(Class clazz, String key); T getVarOrDefault(Class clazz, String key, T defaultValue); diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/exception/BuilderException.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/exception/BuilderException.java similarity index 83% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/exception/BuilderException.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/exception/BuilderException.java index b4f35643aa..b00c15f7e0 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/exception/BuilderException.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/exception/BuilderException.java @@ -15,11 +15,10 @@ * limitations under the License. */ -package org.apache.linkis.cli.core.exception; +package org.apache.linkis.cli.application.exception; -import org.apache.linkis.cli.common.exception.LinkisClientRuntimeException; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.common.exception.error.ErrorMsg; +import org.apache.linkis.cli.application.exception.error.ErrorLevel; +import org.apache.linkis.cli.application.exception.error.ErrorMsg; public class BuilderException extends LinkisClientRuntimeException { private static final long serialVersionUID = 5454234257L; diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/exception/CommandException.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/exception/CommandException.java similarity index 85% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/exception/CommandException.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/exception/CommandException.java index 6f6c5512d4..e0cff74b1e 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/exception/CommandException.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/exception/CommandException.java @@ -15,12 +15,11 @@ * limitations under the License. */ -package org.apache.linkis.cli.core.exception; +package org.apache.linkis.cli.application.exception; -import org.apache.linkis.cli.common.entity.command.CmdType; -import org.apache.linkis.cli.common.exception.LinkisClientRuntimeException; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.common.exception.error.ErrorMsg; +import org.apache.linkis.cli.application.entity.command.CmdType; +import org.apache.linkis.cli.application.exception.error.ErrorLevel; +import org.apache.linkis.cli.application.exception.error.ErrorMsg; public class CommandException extends LinkisClientRuntimeException { private static final long serialVersionUID = 745261661L; diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/exception/LinkisClientException.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/exception/LinkisClientException.java similarity index 94% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/exception/LinkisClientException.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/exception/LinkisClientException.java index 7ffd880b73..9dcb69a410 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/exception/LinkisClientException.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/exception/LinkisClientException.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.linkis.cli.common.exception; +package org.apache.linkis.cli.application.exception; public abstract class LinkisClientException extends Exception { private static final long serialVersionUID = 42563456489L; diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/exception/LinkisClientExecutionException.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/exception/LinkisClientExecutionException.java similarity index 84% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/exception/LinkisClientExecutionException.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/exception/LinkisClientExecutionException.java index e97d068d4e..3acfd71f99 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/exception/LinkisClientExecutionException.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/exception/LinkisClientExecutionException.java @@ -15,11 +15,10 @@ * limitations under the License. */ -package org.apache.linkis.cli.core.exception; +package org.apache.linkis.cli.application.exception; -import org.apache.linkis.cli.common.exception.LinkisClientRuntimeException; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.common.exception.error.ErrorMsg; +import org.apache.linkis.cli.application.exception.error.ErrorLevel; +import org.apache.linkis.cli.application.exception.error.ErrorMsg; public class LinkisClientExecutionException extends LinkisClientRuntimeException { private static final long serialVersionUID = 987189405659L; diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/exception/LinkisClientRuntimeException.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/exception/LinkisClientRuntimeException.java similarity index 94% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/exception/LinkisClientRuntimeException.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/exception/LinkisClientRuntimeException.java index 98538b05e4..0d5f4f7976 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/exception/LinkisClientRuntimeException.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/exception/LinkisClientRuntimeException.java @@ -15,10 +15,10 @@ * limitations under the License. */ -package org.apache.linkis.cli.common.exception; +package org.apache.linkis.cli.application.exception; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.common.exception.error.ErrorMsg; +import org.apache.linkis.cli.application.exception.error.ErrorLevel; +import org.apache.linkis.cli.application.exception.error.ErrorMsg; import java.text.MessageFormat; diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/exception/PresenterException.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/exception/PresenterException.java similarity index 83% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/exception/PresenterException.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/exception/PresenterException.java index dbfd57a2cc..42aa1b1e17 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/exception/PresenterException.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/exception/PresenterException.java @@ -15,11 +15,10 @@ * limitations under the License. */ -package org.apache.linkis.cli.core.exception; +package org.apache.linkis.cli.application.exception; -import org.apache.linkis.cli.common.exception.LinkisClientRuntimeException; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.common.exception.error.ErrorMsg; +import org.apache.linkis.cli.application.exception.error.ErrorLevel; +import org.apache.linkis.cli.application.exception.error.ErrorMsg; public class PresenterException extends LinkisClientRuntimeException { private static final long serialVersionUID = 212314213L; diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/exception/PropsException.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/exception/PropsException.java similarity index 83% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/exception/PropsException.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/exception/PropsException.java index e7e3799dcc..d39a9de3c0 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/exception/PropsException.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/exception/PropsException.java @@ -15,11 +15,10 @@ * limitations under the License. */ -package org.apache.linkis.cli.core.exception; +package org.apache.linkis.cli.application.exception; -import org.apache.linkis.cli.common.exception.LinkisClientRuntimeException; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.common.exception.error.ErrorMsg; +import org.apache.linkis.cli.application.exception.error.ErrorLevel; +import org.apache.linkis.cli.application.exception.error.ErrorMsg; public class PropsException extends LinkisClientRuntimeException { private static final long serialVersionUID = 182747823415933L; diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/exception/TransformerException.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/exception/TransformerException.java similarity index 83% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/exception/TransformerException.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/exception/TransformerException.java index efb578b971..d5b35c6c8d 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/exception/TransformerException.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/exception/TransformerException.java @@ -15,11 +15,10 @@ * limitations under the License. */ -package org.apache.linkis.cli.core.exception; +package org.apache.linkis.cli.application.exception; -import org.apache.linkis.cli.common.exception.LinkisClientRuntimeException; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.common.exception.error.ErrorMsg; +import org.apache.linkis.cli.application.exception.error.ErrorLevel; +import org.apache.linkis.cli.application.exception.error.ErrorMsg; public class TransformerException extends LinkisClientRuntimeException { private static final long serialVersionUID = 5454234257L; diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/exception/UnknownException.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/exception/UnknownException.java similarity index 83% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/exception/UnknownException.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/exception/UnknownException.java index ff5cfac139..a3fecf3537 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/exception/UnknownException.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/exception/UnknownException.java @@ -15,11 +15,10 @@ * limitations under the License. */ -package org.apache.linkis.cli.core.exception; +package org.apache.linkis.cli.application.exception; -import org.apache.linkis.cli.common.exception.LinkisClientRuntimeException; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.common.exception.error.ErrorMsg; +import org.apache.linkis.cli.application.exception.error.ErrorLevel; +import org.apache.linkis.cli.application.exception.error.ErrorMsg; public class UnknownException extends LinkisClientRuntimeException { private static final long serialVersionUID = 974159L; diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/exception/ValidateException.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/exception/ValidateException.java similarity index 83% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/exception/ValidateException.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/exception/ValidateException.java index 95896d9aa0..f796328dd5 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/exception/ValidateException.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/exception/ValidateException.java @@ -15,11 +15,10 @@ * limitations under the License. */ -package org.apache.linkis.cli.core.exception; +package org.apache.linkis.cli.application.exception; -import org.apache.linkis.cli.common.exception.LinkisClientRuntimeException; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.common.exception.error.ErrorMsg; +import org.apache.linkis.cli.application.exception.error.ErrorLevel; +import org.apache.linkis.cli.application.exception.error.ErrorMsg; public class ValidateException extends LinkisClientRuntimeException { private static final long serialVersionUID = 5454234257L; diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/exception/VarAccessException.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/exception/VarAccessException.java similarity index 83% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/exception/VarAccessException.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/exception/VarAccessException.java index 34509947d4..31dc848e36 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/exception/VarAccessException.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/exception/VarAccessException.java @@ -15,11 +15,10 @@ * limitations under the License. */ -package org.apache.linkis.cli.core.exception; +package org.apache.linkis.cli.application.exception; -import org.apache.linkis.cli.common.exception.LinkisClientRuntimeException; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.common.exception.error.ErrorMsg; +import org.apache.linkis.cli.application.exception.error.ErrorLevel; +import org.apache.linkis.cli.application.exception.error.ErrorMsg; public class VarAccessException extends LinkisClientRuntimeException { private static final long serialVersionUID = 125344127L; diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/exception/error/CommonErrMsg.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/exception/error/CommonErrMsg.java similarity index 96% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/exception/error/CommonErrMsg.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/exception/error/CommonErrMsg.java index 67eb0701c7..ea6247e0c4 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/exception/error/CommonErrMsg.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/exception/error/CommonErrMsg.java @@ -15,9 +15,7 @@ * limitations under the License. */ -package org.apache.linkis.cli.core.exception.error; - -import org.apache.linkis.cli.common.exception.error.ErrorMsg; +package org.apache.linkis.cli.application.exception.error; public enum CommonErrMsg implements ErrorMsg { diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/exception/error/ErrorLevel.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/exception/error/ErrorLevel.java similarity index 96% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/exception/error/ErrorLevel.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/exception/error/ErrorLevel.java index ed2867c72d..170cedf9a5 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/exception/error/ErrorLevel.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/exception/error/ErrorLevel.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.linkis.cli.common.exception.error; +package org.apache.linkis.cli.application.exception.error; public enum ErrorLevel { /** warn 1 error 2 fatal 3 */ diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/exception/error/ErrorMsg.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/exception/error/ErrorMsg.java similarity index 93% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/exception/error/ErrorMsg.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/exception/error/ErrorMsg.java index 3d092fe67d..4b6c5d3244 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/exception/error/ErrorMsg.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/exception/error/ErrorMsg.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.linkis.cli.common.exception.error; +package org.apache.linkis.cli.application.exception.error; public interface ErrorMsg { diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/command/LinkisCmdType.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/CliCmdType.java similarity index 78% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/command/LinkisCmdType.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/CliCmdType.java index f03c3e779b..f2a352c542 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/command/LinkisCmdType.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/CliCmdType.java @@ -17,23 +17,23 @@ package org.apache.linkis.cli.application.interactor.command; -import org.apache.linkis.cli.common.entity.command.CmdType; -import org.apache.linkis.cli.core.constants.CommonConstants; +import org.apache.linkis.cli.application.constants.CliConstants; +import org.apache.linkis.cli.application.entity.command.CmdType; -public enum LinkisCmdType implements CmdType { - UNIVERSAL(CommonConstants.UNIVERSAL_SUBCMD, 1, CommonConstants.UNIVERSAL_SUBCMD_DESC); +public enum CliCmdType implements CmdType { + UNIVERSAL(CliConstants.UNIVERSAL_SUBCMD, 1, CliConstants.UNIVERSAL_SUBCMD_DESC); private int id; private String name; private String desc; - LinkisCmdType(String name, int id) { + CliCmdType(String name, int id) { this.id = id; this.name = name; this.desc = null; } - LinkisCmdType(String name, int id, String desc) { + CliCmdType(String name, int id, String desc) { this.id = id; this.name = name; this.desc = desc; diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/CmdTemplateFactory.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/CmdTemplateFactory.java similarity index 83% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/CmdTemplateFactory.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/CmdTemplateFactory.java index 91a9f796cb..f3b901c1f8 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/CmdTemplateFactory.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/CmdTemplateFactory.java @@ -15,13 +15,13 @@ * limitations under the License. */ -package org.apache.linkis.cli.core.interactor.command; +package org.apache.linkis.cli.application.interactor.command; -import org.apache.linkis.cli.common.entity.command.CmdTemplate; -import org.apache.linkis.cli.common.entity.command.CmdType; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.exception.CommandException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.entity.command.CmdTemplate; +import org.apache.linkis.cli.application.entity.command.CmdType; +import org.apache.linkis.cli.application.exception.CommandException; +import org.apache.linkis.cli.application.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.exception.error.ErrorLevel; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/SpecialMap.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/SpecialMap.java similarity index 94% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/SpecialMap.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/SpecialMap.java index 86075e3328..427c35baef 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/SpecialMap.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/SpecialMap.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.linkis.cli.core.interactor.command; +package org.apache.linkis.cli.application.interactor.command; import java.util.HashMap; import java.util.Map; diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/fitter/AbstractFitter.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/fitter/AbstractFitter.java similarity index 89% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/fitter/AbstractFitter.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/fitter/AbstractFitter.java index d32d05bbb6..f5e1a99b58 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/fitter/AbstractFitter.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/fitter/AbstractFitter.java @@ -15,18 +15,18 @@ * limitations under the License. */ -package org.apache.linkis.cli.core.interactor.command.fitter; - -import org.apache.linkis.cli.common.entity.command.CmdOption; -import org.apache.linkis.cli.common.entity.command.CmdTemplate; -import org.apache.linkis.cli.common.entity.command.CmdType; -import org.apache.linkis.cli.common.exception.LinkisClientRuntimeException; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.constants.CommonConstants; -import org.apache.linkis.cli.core.exception.CommandException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; -import org.apache.linkis.cli.core.interactor.command.template.option.Flag; -import org.apache.linkis.cli.core.interactor.command.template.option.Parameter; +package org.apache.linkis.cli.application.interactor.command.fitter; + +import org.apache.linkis.cli.application.constants.CliConstants; +import org.apache.linkis.cli.application.entity.command.CmdOption; +import org.apache.linkis.cli.application.entity.command.CmdTemplate; +import org.apache.linkis.cli.application.entity.command.CmdType; +import org.apache.linkis.cli.application.exception.CommandException; +import org.apache.linkis.cli.application.exception.LinkisClientRuntimeException; +import org.apache.linkis.cli.application.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.exception.error.ErrorLevel; +import org.apache.linkis.cli.application.interactor.command.template.option.Flag; +import org.apache.linkis.cli.application.interactor.command.template.option.Parameter; import org.apache.commons.lang3.StringUtils; @@ -225,7 +225,7 @@ private final int setParameterValue( Parameter param = (Parameter) cmdOption; if (param.accepctArrayValue()) { String[] args2 = Arrays.copyOfRange(args, argIdx, args.length); - param.setValueWithStr(StringUtils.join(args2, CommonConstants.ARRAY_SEQ)); + param.setValueWithStr(StringUtils.join(args2, CliConstants.ARRAY_SEQ)); return args.length; } else { parameters.get(paraIdx).setValueWithStr(args[argIdx]); diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/fitter/Fitter.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/fitter/Fitter.java similarity index 80% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/fitter/Fitter.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/fitter/Fitter.java index 305ffcaef1..cfd0f0fcfa 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/fitter/Fitter.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/fitter/Fitter.java @@ -15,10 +15,10 @@ * limitations under the License. */ -package org.apache.linkis.cli.core.interactor.command.fitter; +package org.apache.linkis.cli.application.interactor.command.fitter; -import org.apache.linkis.cli.common.entity.command.CmdTemplate; -import org.apache.linkis.cli.common.exception.LinkisClientRuntimeException; +import org.apache.linkis.cli.application.entity.command.CmdTemplate; +import org.apache.linkis.cli.application.exception.LinkisClientRuntimeException; public interface Fitter { FitterResult fit(String[] input, CmdTemplate templateCopy) throws LinkisClientRuntimeException; diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/fitter/FitterResult.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/fitter/FitterResult.java similarity index 90% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/fitter/FitterResult.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/fitter/FitterResult.java index 5e9f9820b1..abcee0d9af 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/fitter/FitterResult.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/fitter/FitterResult.java @@ -15,9 +15,9 @@ * limitations under the License. */ -package org.apache.linkis.cli.core.interactor.command.fitter; +package org.apache.linkis.cli.application.interactor.command.fitter; -import org.apache.linkis.cli.common.entity.command.CmdTemplate; +import org.apache.linkis.cli.application.entity.command.CmdTemplate; public class FitterResult { String[] remains; diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/fitter/FitterUtils.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/fitter/FitterUtils.java similarity index 93% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/fitter/FitterUtils.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/fitter/FitterUtils.java index 40d80d84b2..1b6b46cd10 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/fitter/FitterUtils.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/fitter/FitterUtils.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.linkis.cli.core.interactor.command.fitter; +package org.apache.linkis.cli.application.interactor.command.fitter; public class FitterUtils { diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/fitter/SingleTplFitter.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/fitter/SingleTplFitter.java similarity index 79% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/fitter/SingleTplFitter.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/fitter/SingleTplFitter.java index 8eed2ea89f..06b7d7e4f8 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/fitter/SingleTplFitter.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/fitter/SingleTplFitter.java @@ -15,13 +15,13 @@ * limitations under the License. */ -package org.apache.linkis.cli.core.interactor.command.fitter; +package org.apache.linkis.cli.application.interactor.command.fitter; -import org.apache.linkis.cli.common.entity.command.CmdTemplate; -import org.apache.linkis.cli.common.exception.LinkisClientRuntimeException; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.exception.CommandException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.entity.command.CmdTemplate; +import org.apache.linkis.cli.application.exception.CommandException; +import org.apache.linkis.cli.application.exception.LinkisClientRuntimeException; +import org.apache.linkis.cli.application.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.exception.error.ErrorLevel; import java.util.ArrayList; import java.util.List; diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/parser/AbstarctParser.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/parser/AbstarctParser.java similarity index 85% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/parser/AbstarctParser.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/parser/AbstarctParser.java index 30c0e1f014..16107b1869 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/parser/AbstarctParser.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/parser/AbstarctParser.java @@ -15,20 +15,20 @@ * limitations under the License. */ -package org.apache.linkis.cli.core.interactor.command.parser; - -import org.apache.linkis.cli.common.entity.command.CmdOption; -import org.apache.linkis.cli.common.entity.command.CmdTemplate; -import org.apache.linkis.cli.common.entity.command.ParamItem; -import org.apache.linkis.cli.common.entity.command.Params; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.exception.CommandException; -import org.apache.linkis.cli.core.exception.TransformerException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; -import org.apache.linkis.cli.core.interactor.command.SpecialMap; -import org.apache.linkis.cli.core.interactor.command.fitter.Fitter; -import org.apache.linkis.cli.core.interactor.command.parser.result.ParseResult; -import org.apache.linkis.cli.core.interactor.command.parser.transformer.ParamKeyMapper; +package org.apache.linkis.cli.application.interactor.command.parser; + +import org.apache.linkis.cli.application.entity.command.CmdOption; +import org.apache.linkis.cli.application.entity.command.CmdTemplate; +import org.apache.linkis.cli.application.entity.command.ParamItem; +import org.apache.linkis.cli.application.entity.command.Params; +import org.apache.linkis.cli.application.exception.CommandException; +import org.apache.linkis.cli.application.exception.TransformerException; +import org.apache.linkis.cli.application.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.exception.error.ErrorLevel; +import org.apache.linkis.cli.application.interactor.command.SpecialMap; +import org.apache.linkis.cli.application.interactor.command.fitter.Fitter; +import org.apache.linkis.cli.application.interactor.command.parser.result.ParseResult; +import org.apache.linkis.cli.application.interactor.command.parser.transformer.ParamKeyMapper; import org.apache.commons.lang3.StringUtils; diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/parser/Parser.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/parser/Parser.java similarity index 83% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/parser/Parser.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/parser/Parser.java index 0f3c847740..0e10977a63 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/parser/Parser.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/parser/Parser.java @@ -15,10 +15,10 @@ * limitations under the License. */ -package org.apache.linkis.cli.core.interactor.command.parser; +package org.apache.linkis.cli.application.interactor.command.parser; -import org.apache.linkis.cli.common.entity.command.Params; -import org.apache.linkis.cli.core.interactor.command.parser.result.ParseResult; +import org.apache.linkis.cli.application.entity.command.Params; +import org.apache.linkis.cli.application.interactor.command.parser.result.ParseResult; /** * 1. parse cmd arguments and fill into template 2. generate unique identifier for sub command 3. diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/parser/SingleCmdParser.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/parser/SingleCmdParser.java similarity index 77% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/parser/SingleCmdParser.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/parser/SingleCmdParser.java index aea3086fd5..4afb7a759f 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/parser/SingleCmdParser.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/parser/SingleCmdParser.java @@ -15,15 +15,15 @@ * limitations under the License. */ -package org.apache.linkis.cli.core.interactor.command.parser; +package org.apache.linkis.cli.application.interactor.command.parser; -import org.apache.linkis.cli.common.entity.command.CmdTemplate; -import org.apache.linkis.cli.common.entity.command.Params; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.exception.CommandException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; -import org.apache.linkis.cli.core.interactor.command.fitter.FitterResult; -import org.apache.linkis.cli.core.interactor.command.parser.result.ParseResult; +import org.apache.linkis.cli.application.entity.command.CmdTemplate; +import org.apache.linkis.cli.application.entity.command.Params; +import org.apache.linkis.cli.application.exception.CommandException; +import org.apache.linkis.cli.application.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.exception.error.ErrorLevel; +import org.apache.linkis.cli.application.interactor.command.fitter.FitterResult; +import org.apache.linkis.cli.application.interactor.command.parser.result.ParseResult; import java.util.Arrays; diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/parser/result/ParseResult.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/parser/result/ParseResult.java similarity index 88% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/parser/result/ParseResult.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/parser/result/ParseResult.java index 2a528f3b86..75059c86c5 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/parser/result/ParseResult.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/parser/result/ParseResult.java @@ -15,10 +15,10 @@ * limitations under the License. */ -package org.apache.linkis.cli.core.interactor.command.parser.result; +package org.apache.linkis.cli.application.interactor.command.parser.result; -import org.apache.linkis.cli.common.entity.command.CmdTemplate; -import org.apache.linkis.cli.common.entity.command.Params; +import org.apache.linkis.cli.application.entity.command.CmdTemplate; +import org.apache.linkis.cli.application.entity.command.Params; public class ParseResult { CmdTemplate parsedTemplate; diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/parser/transformer/ParamKeyMapper.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/parser/transformer/ParamKeyMapper.java similarity index 88% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/parser/transformer/ParamKeyMapper.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/parser/transformer/ParamKeyMapper.java index 3799906bd5..f162a1c84c 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/parser/transformer/ParamKeyMapper.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/parser/transformer/ParamKeyMapper.java @@ -15,13 +15,13 @@ * limitations under the License. */ -package org.apache.linkis.cli.core.interactor.command.parser.transformer; +package org.apache.linkis.cli.application.interactor.command.parser.transformer; -import org.apache.linkis.cli.common.exception.LinkisClientRuntimeException; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.exception.CommandException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; -import org.apache.linkis.cli.core.utils.CommonUtils; +import org.apache.linkis.cli.application.exception.CommandException; +import org.apache.linkis.cli.application.exception.LinkisClientRuntimeException; +import org.apache.linkis.cli.application.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.exception.error.ErrorLevel; +import org.apache.linkis.cli.application.utils.CliUtils; import org.apache.commons.lang3.StringUtils; @@ -79,7 +79,7 @@ public void updateMapping(String key, String targetKey) { /** update keyMapping according to kv-String. */ private void updateMappingbyConfig(String kvString) { if (StringUtils.isNotBlank(kvString)) { - Map result = CommonUtils.parseKVStringToMap(kvString, ","); + Map result = CliUtils.parseKVStringToMap(kvString, ","); this.mapperRules.putAll(result); } } diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/template/AbstractCmdTemplate.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/template/AbstractCmdTemplate.java similarity index 91% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/template/AbstractCmdTemplate.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/template/AbstractCmdTemplate.java index 992b58effc..ec1f318c2b 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/template/AbstractCmdTemplate.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/template/AbstractCmdTemplate.java @@ -15,20 +15,20 @@ * limitations under the License. */ -package org.apache.linkis.cli.core.interactor.command.template; - -import org.apache.linkis.cli.common.entity.command.CmdOption; -import org.apache.linkis.cli.common.entity.command.CmdTemplate; -import org.apache.linkis.cli.common.entity.command.CmdType; -import org.apache.linkis.cli.common.exception.LinkisClientRuntimeException; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.constants.CommonConstants; -import org.apache.linkis.cli.core.exception.CommandException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; -import org.apache.linkis.cli.core.interactor.command.SpecialMap; -import org.apache.linkis.cli.core.interactor.command.template.option.*; -import org.apache.linkis.cli.core.utils.converter.AbstractStringConverter; -import org.apache.linkis.cli.core.utils.converter.PredefinedStringConverters; +package org.apache.linkis.cli.application.interactor.command.template; + +import org.apache.linkis.cli.application.constants.CliConstants; +import org.apache.linkis.cli.application.entity.command.CmdOption; +import org.apache.linkis.cli.application.entity.command.CmdTemplate; +import org.apache.linkis.cli.application.entity.command.CmdType; +import org.apache.linkis.cli.application.exception.CommandException; +import org.apache.linkis.cli.application.exception.LinkisClientRuntimeException; +import org.apache.linkis.cli.application.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.exception.error.ErrorLevel; +import org.apache.linkis.cli.application.interactor.command.SpecialMap; +import org.apache.linkis.cli.application.interactor.command.template.converter.AbstractStringConverter; +import org.apache.linkis.cli.application.interactor.command.template.converter.PredefinedStringConverters; +import org.apache.linkis.cli.application.interactor.command.template.option.*; import java.lang.reflect.Field; import java.util.*; @@ -291,10 +291,10 @@ protected final SpecialMapOption speciaMapOption( private void checkIllegalOption(final String[] names) { if (names == null || names.length <= 0) { throw new IllegalArgumentException("At least one cmdType should be given to CmdOption."); - } else if (names.length > CommonConstants.MAX_NUM_OF_COMMAND_ARGUEMENTS) { + } else if (names.length > CliConstants.MAX_NUM_OF_COMMAND_ARGUEMENTS) { throw new IllegalArgumentException( "At most " - + CommonConstants.MAX_NUM_OF_COMMAND_ARGUEMENTS + + CliConstants.MAX_NUM_OF_COMMAND_ARGUEMENTS + " cmdType can be given to CmdOption."); } else { for (String name : names) { diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/command/template/UniversalCmdTemplate.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/template/UniversalCmdTemplate.java similarity index 74% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/command/template/UniversalCmdTemplate.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/template/UniversalCmdTemplate.java index e238038aea..f66f5ab386 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/command/template/UniversalCmdTemplate.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/template/UniversalCmdTemplate.java @@ -17,15 +17,14 @@ package org.apache.linkis.cli.application.interactor.command.template; -import org.apache.linkis.cli.application.constants.AppConstants; -import org.apache.linkis.cli.application.constants.AppKeys; -import org.apache.linkis.cli.application.interactor.command.LinkisCmdType; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.exception.CommandException; -import org.apache.linkis.cli.core.exception.ValidateException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; -import org.apache.linkis.cli.core.interactor.command.template.AbstractCmdTemplate; -import org.apache.linkis.cli.core.interactor.command.template.option.*; +import org.apache.linkis.cli.application.constants.CliConstants; +import org.apache.linkis.cli.application.constants.CliKeys; +import org.apache.linkis.cli.application.exception.CommandException; +import org.apache.linkis.cli.application.exception.ValidateException; +import org.apache.linkis.cli.application.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.exception.error.ErrorLevel; +import org.apache.linkis.cli.application.interactor.command.CliCmdType; +import org.apache.linkis.cli.application.interactor.command.template.option.*; import org.apache.commons.lang3.StringUtils; @@ -36,109 +35,118 @@ public class UniversalCmdTemplate extends AbstractCmdTemplate implements Cloneab protected StdOption gatewayUrl = option( - AppKeys.LINKIS_CLIENT_COMMON, - AppKeys.LINKIS_COMMON_GATEWAY_URL, + CliKeys.LINKIS_CLIENT_COMMON, + CliKeys.LINKIS_COMMON_GATEWAY_URL, new String[] {"--gatewayUrl"}, "specify linkis gateway url", true, ""); protected StdOption authenticatationStrategy = option( - AppKeys.LINKIS_CLIENT_COMMON, - AppKeys.LINKIS_COMMON_AUTHENTICATION_STRATEGY, + CliKeys.LINKIS_CLIENT_COMMON, + CliKeys.LINKIS_COMMON_AUTHENTICATION_STRATEGY, new String[] {"--authStg"}, "specify linkis authentication strategy", true, ""); protected StdOption authKey = option( - AppKeys.LINKIS_CLIENT_COMMON, - AppKeys.LINKIS_COMMON_TOKEN_KEY, + CliKeys.LINKIS_CLIENT_COMMON, + CliKeys.LINKIS_COMMON_TOKEN_KEY, new String[] {"--authKey"}, "specify linkis authentication key(tokenKey)", true, ""); protected StdOption authValue = option( - AppKeys.LINKIS_CLIENT_COMMON, - AppKeys.LINKIS_COMMON_TOKEN_VALUE, + CliKeys.LINKIS_CLIENT_COMMON, + CliKeys.LINKIS_COMMON_TOKEN_VALUE, new String[] {"--authVal"}, "specify linkis authentication value(tokenValue)", true, ""); protected StdOption userConfigPath = option( - AppKeys.LINKIS_CLIENT_COMMON, - AppKeys.LINKIS_CLIENT_USER_CONFIG, + CliKeys.LINKIS_CLIENT_COMMON, + CliKeys.LINKIS_CLIENT_USER_CONFIG, new String[] {"--userConf"}, "specify user configuration file path(absolute)", true, ""); protected StdOption killOpt = option( - AppKeys.LINKIS_CLIENT_COMMON, - AppKeys.LINKIS_CLIENT_KILL_OPT, + CliKeys.LINKIS_CLIENT_COMMON, + CliKeys.LINKIS_CLIENT_KILL_OPT, new String[] {"--kill"}, "specify linkis taskId for job to be killed", true, ""); protected StdOption logOpt = option( - AppKeys.LINKIS_CLIENT_COMMON, - AppKeys.LINKIS_CLIENT_LOG_OPT, + CliKeys.LINKIS_CLIENT_COMMON, + CliKeys.LINKIS_CLIENT_LOG_OPT, new String[] {"--log"}, "specify linkis taskId for querying job status", true, ""); protected StdOption resultOpt = option( - AppKeys.LINKIS_CLIENT_COMMON, - AppKeys.LINKIS_CLIENT_RESULT_OPT, + CliKeys.LINKIS_CLIENT_COMMON, + CliKeys.LINKIS_CLIENT_RESULT_OPT, new String[] {"--result"}, "specify linkis taskId for querying job status", true, ""); protected StdOption statusOpt = option( - AppKeys.LINKIS_CLIENT_COMMON, - AppKeys.LINKIS_CLIENT_STATUS_OPT, + CliKeys.LINKIS_CLIENT_COMMON, + CliKeys.LINKIS_CLIENT_STATUS_OPT, new String[] {"--status"}, "specify linkis taskId for querying job status", true, ""); protected StdOption asyncOpt = option( - AppKeys.LINKIS_CLIENT_COMMON, - AppKeys.LINKIS_CLIENT_ASYNC_OPT, + CliKeys.LINKIS_CLIENT_COMMON, + CliKeys.LINKIS_CLIENT_ASYNC_OPT, new String[] {"--async"}, "specify linkis taskId for querying job status", true, false); protected StdOption modeOpt = option( - AppKeys.LINKIS_CLIENT_COMMON, - AppKeys.LINKIS_CLIENT_MODE_OPT, + CliKeys.LINKIS_CLIENT_COMMON, + CliKeys.LINKIS_CLIENT_MODE_OPT, new String[] {"--mode"}, "specify linkis execution mode: " - + AppConstants.UJES_MODE + + CliConstants.UJES_MODE + "/" - + AppConstants.ONCE_MODE + + CliConstants.ONCE_MODE + ".", true, - AppConstants.UJES_MODE); + CliConstants.UJES_MODE); protected Flag helpOpt = flag( - AppKeys.LINKIS_CLIENT_COMMON, - AppKeys.LINKIS_CLIENT_HELP_OPT, + CliKeys.LINKIS_CLIENT_COMMON, + CliKeys.LINKIS_CLIENT_HELP_OPT, new String[] {"--help"}, "specify linkis taskId for querying job status", true, false); + protected StdOption clusterOP = + option( + CliKeys.JOB_LABEL, + CliKeys.JOB_LABEL_CLUSTER, + new String[] {"-yarnCluster"}, + "specify linkis yarn cluster for this job", + true, + ""); + protected StdOption engineTypeOP = option( - AppKeys.JOB_LABEL, - AppKeys.JOB_LABEL_ENGINE_TYPE, + CliKeys.JOB_LABEL, + CliKeys.JOB_LABEL_ENGINE_TYPE, new String[] {"-engineType"}, "specify linkis engineType for this job", true, @@ -146,16 +154,16 @@ public class UniversalCmdTemplate extends AbstractCmdTemplate implements Cloneab protected StdOption codeTypeOp = option( - AppKeys.JOB_LABEL, - AppKeys.JOB_LABEL_CODE_TYPE, + CliKeys.JOB_LABEL, + CliKeys.JOB_LABEL_CODE_TYPE, new String[] {"-codeType"}, "specify linkis runType for this job", true, ""); protected StdOption codePathOp = option( - AppKeys.LINKIS_CLIENT_COMMON, - AppKeys.JOB_COMMON_CODE_PATH, + CliKeys.LINKIS_CLIENT_COMMON, + CliKeys.JOB_COMMON_CODE_PATH, new String[] {"-codePath"}, "specify file path that contains code you want to execute", true, @@ -163,8 +171,8 @@ public class UniversalCmdTemplate extends AbstractCmdTemplate implements Cloneab protected StdOption codeOp = option( - AppKeys.JOB_EXEC, - AppKeys.JOB_EXEC_CODE, + CliKeys.JOB_EXEC, + CliKeys.JOB_EXEC_CODE, new String[] {"-code"}, "specify code that you want to execute", true, @@ -172,8 +180,8 @@ public class UniversalCmdTemplate extends AbstractCmdTemplate implements Cloneab protected StdOption scriptPathOp = option( - AppKeys.JOB_SOURCE, - AppKeys.JOB_SOURCE_SCRIPT_PATH, + CliKeys.JOB_SOURCE, + CliKeys.JOB_SOURCE_SCRIPT_PATH, new String[] {"-scriptPath"}, "specify remote path for your uploaded script", true, @@ -181,8 +189,8 @@ public class UniversalCmdTemplate extends AbstractCmdTemplate implements Cloneab protected StdOption submitUser = option( - AppKeys.LINKIS_CLIENT_COMMON, - AppKeys.JOB_COMMON_SUBMIT_USER, + CliKeys.LINKIS_CLIENT_COMMON, + CliKeys.JOB_COMMON_SUBMIT_USER, new String[] {"-submitUser"}, "specify submit user for this job", true, @@ -190,8 +198,8 @@ public class UniversalCmdTemplate extends AbstractCmdTemplate implements Cloneab protected StdOption proxyUser = option( - AppKeys.LINKIS_CLIENT_COMMON, - AppKeys.JOB_COMMON_PROXY_USER, + CliKeys.LINKIS_CLIENT_COMMON, + CliKeys.JOB_COMMON_PROXY_USER, new String[] {"-proxyUser"}, "specify proxy user who executes your code in Linkis server-side", true, @@ -199,8 +207,8 @@ public class UniversalCmdTemplate extends AbstractCmdTemplate implements Cloneab protected StdOption creatorOp = option( - AppKeys.LINKIS_CLIENT_COMMON, - AppKeys.JOB_COMMON_CREATOR, + CliKeys.LINKIS_CLIENT_COMMON, + CliKeys.JOB_COMMON_CREATOR, new String[] {"-creator"}, "specify creator for this job", true, @@ -208,8 +216,8 @@ public class UniversalCmdTemplate extends AbstractCmdTemplate implements Cloneab protected StdOption outPathOp = option( - AppKeys.LINKIS_CLIENT_COMMON, - AppKeys.LINKIS_CLIENT_COMMON_OUTPUT_PATH, + CliKeys.LINKIS_CLIENT_COMMON, + CliKeys.LINKIS_CLIENT_COMMON_OUTPUT_PATH, new String[] {"-outPath"}, "specify output path for resultSet. If not specified, then output reset to screen(stdout)", true, @@ -217,67 +225,79 @@ public class UniversalCmdTemplate extends AbstractCmdTemplate implements Cloneab protected MapOption confMapOp = mapOption( - AppKeys.JOB_PARAM_CONF, - AppKeys.JOB_PARAM_CONF, + CliKeys.JOB_PARAM_CONF, + CliKeys.JOB_PARAM_CONF, new String[] {"-confMap"}, "specify configurationMap(startupMap) for your job. You can put any start-up parameters into this Map(e.g. spark.executor.instances). Input format: -confMap key1=value1 -confMap key2=value2", true); protected MapOption runtimeMapOp = mapOption( - AppKeys.JOB_PARAM_RUNTIME, - AppKeys.JOB_PARAM_RUNTIME, + CliKeys.JOB_PARAM_RUNTIME, + CliKeys.JOB_PARAM_RUNTIME, new String[] {"-runtimeMap"}, "specify runtimeMap for your job. You can put any start-up parameters into this Map(e.g. spark.executor.instances). Input format: -runtimeMap key1=value1 -runtimeMap key2=value2", true); protected SpecialMapOption varMapOp = speciaMapOption( - AppKeys.JOB_PARAM_VAR, - AppKeys.JOB_PARAM_VAR, + CliKeys.JOB_PARAM_VAR, + CliKeys.JOB_PARAM_VAR, new String[] {"-varMap"}, "specify variables map. Variables is for key-word substitution. Use \'${key}\' to specify key-word. Input substitution rule as follow: -varMap key1=value1 -varMap key2=value2", true); protected MapOption labelMapOp = mapOption( - AppKeys.JOB_LABEL, - AppKeys.JOB_LABEL, + CliKeys.JOB_LABEL, + CliKeys.JOB_LABEL, new String[] {"-labelMap"}, "specify label map. You can put any Linkis into this Map. Input format: -labelMap labelName1=labelValue1 -labelMap labelName2=labelValue2", true); protected MapOption sourceMapOp = mapOption( - AppKeys.JOB_SOURCE, - AppKeys.JOB_SOURCE, + CliKeys.JOB_SOURCE, + CliKeys.JOB_SOURCE, new String[] {"-sourceMap"}, "specify source map. Input format: -sourceMap key1=value1 -sourceMap key2=value2", true); protected MapOption jobContentMapOp = mapOption( - AppKeys.JOB_CONTENT, - AppKeys.JOB_CONTENT, + CliKeys.JOB_CONTENT, + CliKeys.JOB_CONTENT, new String[] {"-jobContentMap"}, "specify jobContent map. Input format: -jobContentMap key1=value1 -jobContentMap key2=value2", true); + protected Flag versionFlag = + flag( + CliKeys.VERSION, + CliKeys.VERSION, + new String[] {"--version"}, + "show version", + true, + false); + protected Parameter argumentsParas = parameter( - AppKeys.LINKIS_CLIENT_COMMON, - AppKeys.JOB_EXTRA_ARGUMENTS, + CliKeys.LINKIS_CLIENT_COMMON, + CliKeys.JOB_EXTRA_ARGUMENTS, "arguments", "specify arguments if exist any", true, new String[] {""}); public UniversalCmdTemplate() { - super(LinkisCmdType.UNIVERSAL); + super(CliCmdType.UNIVERSAL); } @Override public void checkParams() throws CommandException { + if (versionFlag.hasVal()) { + return; + } int cnt = 0; if (statusOpt.hasVal()) { cnt++; @@ -349,7 +369,7 @@ public void checkParams() throws CommandException { cnt2++; } if (!modeOpt.hasVal() - || StringUtils.equalsIgnoreCase(modeOpt.getValue(), AppConstants.UJES_MODE)) { + || StringUtils.equalsIgnoreCase(modeOpt.getValue(), CliConstants.UJES_MODE)) { if (cnt2 > 1) { throw new ValidateException( "VLD0001", diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/utils/converter/AbstractStringConverter.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/template/converter/AbstractStringConverter.java similarity index 91% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/utils/converter/AbstractStringConverter.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/template/converter/AbstractStringConverter.java index 3d5afaa172..68f3fae396 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/utils/converter/AbstractStringConverter.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/template/converter/AbstractStringConverter.java @@ -15,6 +15,6 @@ * limitations under the License. */ -package org.apache.linkis.cli.core.utils.converter; +package org.apache.linkis.cli.application.interactor.command.template.converter; public abstract class AbstractStringConverter implements Converter {} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/utils/converter/Converter.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/template/converter/Converter.java similarity index 91% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/utils/converter/Converter.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/template/converter/Converter.java index 327f44cf2e..757d73d178 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/utils/converter/Converter.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/template/converter/Converter.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.linkis.cli.core.utils.converter; +package org.apache.linkis.cli.application.interactor.command.template.converter; public interface Converter { TO convert(FROM from); diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/utils/converter/PredefinedStringConverters.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/template/converter/PredefinedStringConverters.java similarity index 92% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/utils/converter/PredefinedStringConverters.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/template/converter/PredefinedStringConverters.java index 5883f68e1b..b6e65c1e33 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/utils/converter/PredefinedStringConverters.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/template/converter/PredefinedStringConverters.java @@ -15,13 +15,13 @@ * limitations under the License. */ -package org.apache.linkis.cli.core.utils.converter; +package org.apache.linkis.cli.application.interactor.command.template.converter; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.constants.CommonConstants; -import org.apache.linkis.cli.core.exception.CommandException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; -import org.apache.linkis.cli.core.interactor.command.SpecialMap; +import org.apache.linkis.cli.application.constants.CliConstants; +import org.apache.linkis.cli.application.exception.CommandException; +import org.apache.linkis.cli.application.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.exception.error.ErrorLevel; +import org.apache.linkis.cli.application.interactor.command.SpecialMap; import org.apache.commons.lang3.StringUtils; @@ -50,7 +50,7 @@ public String[] convert(String from) { if (StringUtils.isBlank(from)) { return null; } - String[] ret = from.trim().split(CommonConstants.ARRAY_SEQ); + String[] ret = from.trim().split(CliConstants.ARRAY_SEQ); for (int i = 0; i < ret.length; i++) { ret[i] = StringUtils.strip(ret[i], " \""); } diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/template/option/BaseOption.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/template/option/BaseOption.java similarity index 93% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/template/option/BaseOption.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/template/option/BaseOption.java index 8e9f288567..e497401bef 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/template/option/BaseOption.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/template/option/BaseOption.java @@ -15,10 +15,10 @@ * limitations under the License. */ -package org.apache.linkis.cli.core.interactor.command.template.option; +package org.apache.linkis.cli.application.interactor.command.template.option; -import org.apache.linkis.cli.common.entity.command.CmdOption; -import org.apache.linkis.cli.core.utils.converter.AbstractStringConverter; +import org.apache.linkis.cli.application.entity.command.CmdOption; +import org.apache.linkis.cli.application.interactor.command.template.converter.AbstractStringConverter; import org.apache.commons.lang3.StringUtils; diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/template/option/Flag.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/template/option/Flag.java similarity index 91% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/template/option/Flag.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/template/option/Flag.java index 8b153fb7ac..ee66a64463 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/template/option/Flag.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/template/option/Flag.java @@ -15,9 +15,9 @@ * limitations under the License. */ -package org.apache.linkis.cli.core.interactor.command.template.option; +package org.apache.linkis.cli.application.interactor.command.template.option; -import org.apache.linkis.cli.core.utils.converter.PredefinedStringConverters; +import org.apache.linkis.cli.application.interactor.command.template.converter.PredefinedStringConverters; import org.apache.commons.lang3.StringUtils; diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/template/option/MapOption.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/template/option/MapOption.java similarity index 93% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/template/option/MapOption.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/template/option/MapOption.java index 095004c4d1..d330d11c9b 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/template/option/MapOption.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/template/option/MapOption.java @@ -15,11 +15,11 @@ * limitations under the License. */ -package org.apache.linkis.cli.core.interactor.command.template.option; +package org.apache.linkis.cli.application.interactor.command.template.option; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.exception.CommandException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.exception.CommandException; +import org.apache.linkis.cli.application.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.exception.error.ErrorLevel; import org.apache.commons.lang3.StringUtils; diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/template/option/Parameter.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/template/option/Parameter.java similarity index 93% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/template/option/Parameter.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/template/option/Parameter.java index 09422106c8..802f451ebb 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/template/option/Parameter.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/template/option/Parameter.java @@ -15,9 +15,9 @@ * limitations under the License. */ -package org.apache.linkis.cli.core.interactor.command.template.option; +package org.apache.linkis.cli.application.interactor.command.template.option; -import org.apache.linkis.cli.core.utils.converter.AbstractStringConverter; +import org.apache.linkis.cli.application.interactor.command.template.converter.AbstractStringConverter; import org.apache.commons.lang3.StringUtils; diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/template/option/SpecialMapOption.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/template/option/SpecialMapOption.java similarity index 90% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/template/option/SpecialMapOption.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/template/option/SpecialMapOption.java index d14758206b..3fc6385db0 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/template/option/SpecialMapOption.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/template/option/SpecialMapOption.java @@ -15,9 +15,9 @@ * limitations under the License. */ -package org.apache.linkis.cli.core.interactor.command.template.option; +package org.apache.linkis.cli.application.interactor.command.template.option; -import org.apache.linkis.cli.core.interactor.command.SpecialMap; +import org.apache.linkis.cli.application.interactor.command.SpecialMap; /** kv-pairs in SpecialMapOption will be excluded by varAccess */ public class SpecialMapOption extends MapOption implements Cloneable { diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/template/option/StdOption.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/template/option/StdOption.java similarity index 92% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/template/option/StdOption.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/template/option/StdOption.java index a9d29aef26..737cd73423 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/template/option/StdOption.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/template/option/StdOption.java @@ -15,9 +15,9 @@ * limitations under the License. */ -package org.apache.linkis.cli.core.interactor.command.template.option; +package org.apache.linkis.cli.application.interactor.command.template.option; -import org.apache.linkis.cli.core.utils.converter.AbstractStringConverter; +import org.apache.linkis.cli.application.interactor.command.template.converter.AbstractStringConverter; import org.apache.commons.lang3.StringUtils; diff --git a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/context/CliCtxImpl.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/context/CliCtxImpl.java new file mode 100644 index 0000000000..7d4f5d94b8 --- /dev/null +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/context/CliCtxImpl.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.cli.application.interactor.context; + +import org.apache.linkis.cli.application.entity.command.CmdTemplate; +import org.apache.linkis.cli.application.entity.command.CmdType; +import org.apache.linkis.cli.application.entity.context.CliCtx; +import org.apache.linkis.cli.application.entity.var.VarAccess; + +import java.util.Map; + +public class CliCtxImpl implements CliCtx { + private CmdType cmdType; + private CmdTemplate cmdTemplate; + private VarAccess varAccess; + private Map extraMap; + + public CliCtxImpl( + CmdType cmdType, CmdTemplate cmdTemplate, VarAccess varAccess, Map extraMap) { + this.cmdType = cmdType; + this.cmdTemplate = cmdTemplate; + this.varAccess = varAccess; + this.extraMap = extraMap; + } + + @Override + public CmdType getCmdType() { + return cmdType; + } + + @Override + public CmdTemplate getTemplate() { + return cmdTemplate; + } + + @Override + public VarAccess getVarAccess() { + return varAccess; + } + + @Override + public Map getExtraMap() { + return extraMap; + } +} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/common/JobKiller.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/common/JobKiller.java new file mode 100644 index 0000000000..6693abe85f --- /dev/null +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/common/JobKiller.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.cli.application.interactor.job.common; + +import org.apache.linkis.cli.application.constants.CliConstants; +import org.apache.linkis.cli.application.operator.ujes.LinkisJobOper; +import org.apache.linkis.cli.application.operator.ujes.LinkisOperResultAdapter; +import org.apache.linkis.cli.application.utils.CliUtils; +import org.apache.linkis.cli.application.utils.LoggerManager; + +import org.apache.commons.lang3.exception.ExceptionUtils; + +import java.text.MessageFormat; +import java.util.HashMap; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class JobKiller { + private static final Logger logger = LoggerFactory.getLogger(JobKiller.class); + + private LinkisJobOper oper; + + public JobKiller(LinkisJobOper linkisJobOperator) { + this.oper = linkisJobOperator; + } + + public KillResult doKill(String username, String jobId) { + + LinkisOperResultAdapter jobInfoResult; + try { + jobInfoResult = oper.queryJobInfo(username, jobId); + } catch (Exception e) { + return new KillResult( + true, "Failed to query jobInfo" + ExceptionUtils.getStackTrace(e), new HashMap<>()); + } + if (jobInfoResult.getUser() == null || jobInfoResult.getJobID() == null) { + return new KillResult(false, "user or jobID is null", new HashMap<>()); + } + if (jobInfoResult.getJobStatus() == null) { + return new KillResult(false, "jobStatus is null", new HashMap<>()); + } + if (jobInfoResult.getJobStatus().isJobCancelled()) { + String msg = "Kill job aborted: Job has already been canceled."; + return new KillResult(false, msg, new HashMap<>()); + } else if (jobInfoResult.getJobStatus().isJobFinishedState()) { + String msg = "Kill job aborted: Job is already in finished-state(SUCCEED/FAILED)."; + return new KillResult(false, msg, new HashMap<>()); + // throw new LinkisClientExecutionException(JobStatus.FAILED, "EXE0004", + // ErrorLevel.ERROR, CommonErrMsg.ExecutionErr, msg); + } else { + try { + LinkisOperResultAdapter jobKillResult = + oper.kill( + jobInfoResult.getUser(), + jobInfoResult.getJobID(), + jobInfoResult.getStrongerExecId()); + } catch (Exception e) { + return new KillResult( + false, + "Exception thrown when trying to send kill request. Messgae: " + + ExceptionUtils.getStackTrace(e), + new HashMap<>()); + } + String msg = "Kill request has been sent"; + LoggerManager.getPlaintTextLogger().info(msg); + int retryCnt = 0; + final int MAX_RETRY = 5 * 6; + while (!jobInfoResult.getJobStatus().isJobFinishedState() + && !jobInfoResult.getJobStatus().isJobCancelled()) { + CliUtils.doSleepQuietly(CliConstants.JOB_QUERY_SLEEP_MILLS); + try { + jobInfoResult = oper.queryJobInfo(jobInfoResult.getUser(), jobInfoResult.getJobID()); + retryCnt = 0; // if exception then will not go here + } catch (Exception e) { + retryCnt++; + CliUtils.doSleepQuietly(5 * CliConstants.JOB_QUERY_SLEEP_MILLS); + if (retryCnt >= MAX_RETRY) { + return new KillResult( + false, + MessageFormat.format( + "After send kill. Client cannot get jobStatus from server continuously for {0} seconds. Client aborted. Assume kill failed! Error message: \n", + MAX_RETRY * 5 * CliConstants.JOB_QUERY_SLEEP_MILLS), + new HashMap<>()); + } + } + } + if (jobInfoResult.getJobStatus().isJobFinishedState() + && !jobInfoResult.getJobStatus().isJobCancelled()) { + msg = "Kill Failed: Job Current status: " + jobInfoResult.getJobStatus(); + return new KillResult(false, msg, new HashMap<>()); + // throw new LinkisClientExecutionException(JobStatus.FAILED, + // "EXE0004", ErrorLevel.ERROR, CommonErrMsg.ExecutionErr, msg); + } else if (jobInfoResult.getJobStatus().isJobCancelled()) { + msg = + MessageFormat.format( + "Kill successful: jobId={0}, status={1}.", + jobInfoResult.getJobID(), jobInfoResult.getJobStatus()); + return new KillResult(true, msg, new HashMap<>()); + // LogUtils.getPlaintTextLogger().info(msg); + } else { + return new KillResult(false, "Unknown Error!!", new HashMap<>()); + } + } + } +} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/builder/ProcessKeyUtils.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/common/KeyParser.java similarity index 85% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/builder/ProcessKeyUtils.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/common/KeyParser.java index 9527a8f513..01295ce676 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/builder/ProcessKeyUtils.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/common/KeyParser.java @@ -15,27 +15,27 @@ * limitations under the License. */ -package org.apache.linkis.cli.application.interactor.job.builder; +package org.apache.linkis.cli.application.interactor.job.common; -import org.apache.linkis.cli.application.constants.AppKeys; +import org.apache.linkis.cli.application.constants.CliKeys; import org.apache.commons.lang3.StringUtils; import java.util.HashMap; import java.util.Map; -public class ProcessKeyUtils { +public class KeyParser { public static Map removePrefixForKeysInMap(Map map) { final String[] PREFIX = new String[] { - AppKeys.JOB_PARAM_CONF, - AppKeys.JOB_PARAM_RUNTIME, - AppKeys.JOB_PARAM_VAR, - AppKeys.JOB_EXEC, - AppKeys.JOB_SOURCE, - AppKeys.JOB_LABEL, - AppKeys.JOB_CONTENT + CliKeys.JOB_PARAM_CONF, + CliKeys.JOB_PARAM_RUNTIME, + CliKeys.JOB_PARAM_VAR, + CliKeys.JOB_EXEC, + CliKeys.JOB_SOURCE, + CliKeys.JOB_LABEL, + CliKeys.JOB_CONTENT }; for (String prefix : PREFIX) { map = removePrefixForKeysInMap(map, prefix); diff --git a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/common/KillResult.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/common/KillResult.java new file mode 100644 index 0000000000..732cd00ec7 --- /dev/null +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/common/KillResult.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.cli.application.interactor.job.common; + +import org.apache.linkis.cli.application.entity.job.JobResult; + +import java.util.Map; + +public class KillResult implements JobResult { + private Boolean success; + private String message; + private Map extraMessage; + + public KillResult(Boolean success, String message, Map extraMessage) { + this.success = success; + this.message = message; + this.extraMessage = extraMessage; + } + + @Override + public Boolean isSuccess() { + return success; + } + + @Override + public String getMessage() { + return message; + } + + public void setMessage(String message) { + this.message = message; + } + + @Override + public Map getExtraMessage() { + return extraMessage; + } + + public void setExtraMessage(Map extraMessage) { + this.extraMessage = extraMessage; + } + + public void setSuccess(Boolean success) { + this.success = success; + } +} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/LinkisJobStatus.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/common/LinkisJobStatus.java similarity index 97% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/LinkisJobStatus.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/common/LinkisJobStatus.java index 9b1435da2a..dd4df9ee9b 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/LinkisJobStatus.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/common/LinkisJobStatus.java @@ -15,10 +15,10 @@ * limitations under the License. */ -package org.apache.linkis.cli.application.interactor.job; +package org.apache.linkis.cli.application.interactor.job.common; +import org.apache.linkis.cli.application.entity.job.JobStatus; import org.apache.linkis.cli.application.operator.once.LinkisNodeStatus; -import org.apache.linkis.cli.common.entity.job.JobStatus; import org.apache.commons.lang3.StringUtils; diff --git a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/common/LogData.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/common/LogData.java new file mode 100644 index 0000000000..5397bd04f1 --- /dev/null +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/common/LogData.java @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.cli.application.interactor.job.common; + +import org.apache.linkis.cli.application.operator.ujes.LinkisOperResultAdapter; + +import java.util.LinkedList; +import java.util.List; +import java.util.concurrent.LinkedBlockingDeque; + +public class LogData { + private String user; + private String jobId; + private String execId; + // log will be fetched and stored in LinkedBlockingDeque, while logRetriever consumes log in + // another thread + private LinkedBlockingDeque logBuffer = new LinkedBlockingDeque(); + private String logPath; // remote path for job log + private Integer nextLogLineIdx; // index of next log line to be fetched + private Boolean hasNextLogLine; // if there is still log to be retrieve + private volatile Boolean logFin = false; // if all log is finished + + public LogData(String user, String jobId, String execId) { + this.user = user; + this.jobId = jobId; + this.execId = execId; + } + + public String getUser() { + return user; + } + + public String getJobID() { + return jobId; + } + + public String getExecID() { + return execId; + } + + public final String getLogPath() { + return logPath; + } + + public final void setLogPath(String logPath) { + this.logPath = logPath; + } + + public String consumeLog() { + List logs = new LinkedList<>(); + this.logBuffer.drainTo(logs, this.logBuffer.size()); + StringBuilder tmp = new StringBuilder(); + for (String str : logs) { + tmp.append(str); + } + return tmp.toString(); + } + + public void appendLog(String log) { + this.logBuffer.add(log); + } + + public Integer getNextLogLineIdx() { + return nextLogLineIdx; + } + + public void setNextLogLineIdx(Integer nextLogLineIdx) { + this.nextLogLineIdx = nextLogLineIdx; + } + + public Boolean hasNextLogLine() { + return hasNextLogLine; + } + + public void setHasNextLogLine(Boolean hasNextLogLine) { + this.hasNextLogLine = hasNextLogLine; + } + + public void updateLog(LinkisOperResultAdapter adapter) { + if (adapter.getLogPath() != null) { + setLogPath(adapter.getLogPath()); + } + if (adapter.getLog() != null + && adapter.getNextLogLine() != null + && adapter.hasNextLogLine() != null) { + setNextLogLineIdx(adapter.getNextLogLine()); + setHasNextLogLine(adapter.hasNextLogLine()); + appendLog(adapter.getLog()); + } + } + + public void setLogFin() { + this.logFin = true; + } + + public Boolean isLogFin() { + return logFin; + } +} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/common/LogRetriever.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/common/LogRetriever.java new file mode 100644 index 0000000000..33943f1748 --- /dev/null +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/common/LogRetriever.java @@ -0,0 +1,202 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.cli.application.interactor.job.common; + +import org.apache.linkis.cli.application.constants.CliConstants; +import org.apache.linkis.cli.application.exception.LinkisClientExecutionException; +import org.apache.linkis.cli.application.exception.LinkisClientRuntimeException; +import org.apache.linkis.cli.application.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.exception.error.ErrorLevel; +import org.apache.linkis.cli.application.observer.event.LinkisClientEvent; +import org.apache.linkis.cli.application.observer.event.LogStartEvent; +import org.apache.linkis.cli.application.observer.listener.LinkisClientListener; +import org.apache.linkis.cli.application.operator.ujes.LinkisJobOper; +import org.apache.linkis.cli.application.operator.ujes.LinkisOperResultAdapter; +import org.apache.linkis.cli.application.utils.CliUtils; +import org.apache.linkis.cli.application.utils.LoggerManager; +import org.apache.linkis.cli.application.utils.SchedulerManager; + +import java.text.MessageFormat; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class LogRetriever { + private static final Logger logger = LoggerFactory.getLogger(LogRetriever.class); + + private LinkisJobOper linkisJobOperator; + private LogData logData; + + private Boolean incLogMode; + + private LinkisClientListener logListener; + private LinkisClientEvent logStartEvent = new LogStartEvent(); + + public LogRetriever( + String user, + String jobId, + String execId, + Boolean incLogMode, + LinkisJobOper linkisJobOperator, + LinkisClientListener logListener) { + this.linkisJobOperator = linkisJobOperator; + this.logListener = logListener; + this.incLogMode = incLogMode; + this.logData = new LogData(user, jobId, execId); + registerLogListener(logListener); + } + + public void retrieveLogAsync() { + if (logData.getUser() == null || logData.getJobID() == null) { + throw new LinkisClientExecutionException( + "EXE0036", ErrorLevel.ERROR, CommonErrMsg.ExecutionErr, "user or jobID is null"); + } + try { + Thread logConsumer = new Thread(() -> notifyLogListener(), "Log-Consumer"); + Thread logRetriever = new Thread(() -> queryLogLoop(logData), "Log-Retriever"); + SchedulerManager.getCachedThreadPoolExecutor().execute(logRetriever); + SchedulerManager.getCachedThreadPoolExecutor().execute(logConsumer); + } catch (Exception e) { + logger.warn("Failed to retrieve log", e); + } + } + + public void waitIncLogComplete() { + int retry = 0; + int MAX_RETRY = 300; // wait for 10 minutes after job finish + while (retry++ < MAX_RETRY) { + if (this.logFinReceived()) { + return; + } + CliUtils.doSleepQuietly(CliConstants.JOB_QUERY_SLEEP_MILLS); + } + String msg = + "Job is in Finished state(SUCCEED/FAILED/CANCELED) but client keep querying inclog for " + + (MAX_RETRY * CliConstants.JOB_QUERY_SLEEP_MILLS / 1000) + + "seconds. Execution ends forcefully. Next will try handle execution result."; + logger.warn(msg); + LoggerManager.getInformationLogger().warn(msg); + } + + public void queryLogLoop(LogData data) { + int curLogIdx; + int nextLogIdx; + boolean hasNext = true; + int retryCnt = 0; + final int MAX_RETRY = 12; // continues fails for 90s, then exit thread + try { + while (hasNext) { + curLogIdx = data.getNextLogLineIdx() == null ? 0 : data.getNextLogLineIdx(); + try { + queryJobLogFromLine(data, curLogIdx); + } catch (Exception e) { + logger.error("Cannot get inc-log:", e); + // and yes sometimes server may not be able to prepare persisted-log + retryCnt++; + if (retryCnt >= MAX_RETRY) { + logger.error( + "Continuously failing to query inc-log for " + + MAX_RETRY * (MAX_RETRY + 2) * 500 / 1000 + + "s. Will no longer try to query log", + e); + break; + } + CliUtils.doSleepQuietly(500l + 500l * retryCnt); // maybe server problem. sleep longer + continue; + } + retryCnt = 0; + nextLogIdx = data.getNextLogLineIdx() == null ? curLogIdx : data.getNextLogLineIdx(); + if (incLogMode) { + hasNext = data.hasNextLogLine() == null ? curLogIdx < nextLogIdx : data.hasNextLogLine(); + } else { + hasNext = curLogIdx < nextLogIdx; + } + if (curLogIdx >= nextLogIdx) { + String msg = + MessageFormat.format( + "Retrieving log, hasNext={0}, nextLogIdx={1}", hasNext, nextLogIdx); + logger.info(msg); + } + CliUtils.doSleepQuietly(CliConstants.JOB_QUERY_SLEEP_MILLS); + } + } catch (Exception e) { + logger.error("Something goes wrong. Job Log may be incomplete", e); + } finally { + sendLogFin(); + } + } + + private void queryJobLogFromLine(LogData data, int fromLine) throws LinkisClientRuntimeException { + + LinkisOperResultAdapter jobInfoResult = + linkisJobOperator.queryJobInfo(data.getUser(), data.getJobID()); + data.updateLog(jobInfoResult); + if (!jobInfoResult.getJobStatus().isJobFinishedState()) { + try { + data.updateLog( + linkisJobOperator.queryRunTimeLogFromLine( + data.getUser(), data.getJobID(), data.getExecID(), fromLine)); + } catch (Exception e) { + // job is finished while we start query log(but request is not send). + // then probably server cache is gone and we got a exception here. + // however we cannot know if this happens based on the exception message + logger.warn( + "Caught exception when querying runtime-log. Probably server-side has close stream. Will try openLog api if Job is completed.", + e); + if (jobInfoResult.getJobStatus().isJobFinishedState()) { + CliUtils.doSleepQuietly(500l); + data.updateLog( + linkisJobOperator.queryPersistedLogFromLine( + data.getLogPath(), data.getUser(), data.getJobID(), fromLine)); + } + } + } else { + try { + data.updateLog( + linkisJobOperator.queryPersistedLogFromLine( + data.getLogPath(), data.getUser(), data.getJobID(), fromLine)); + } catch (Exception e) { + logger.error("Cannot get persisted-inc-log:", e); + // and yes sometimes server may not be able to prepare persisted-log + throw e; + } + } + } + + public Boolean isIncLogMode() { + return incLogMode; + } + + public void registerLogListener(LinkisClientListener observer) { + this.logStartEvent.register(observer); + } + + public void notifyLogListener() { + if (this.logStartEvent.isRegistered()) { + logStartEvent.notifyObserver(logStartEvent, this.logData); + } + } + + public void sendLogFin() { + this.logData.setLogFin(); + } + + public boolean logFinReceived() { + return this.logData.isLogFin(); + } +} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/common/ResultData.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/common/ResultData.java new file mode 100644 index 0000000000..8c9bec028b --- /dev/null +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/common/ResultData.java @@ -0,0 +1,165 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.cli.application.interactor.job.common; + +import org.apache.linkis.cli.application.entity.job.JobStatus; +import org.apache.linkis.cli.application.operator.ujes.LinkisOperResultAdapter; + +import java.util.LinkedList; +import java.util.List; +import java.util.concurrent.LinkedBlockingDeque; + +public class ResultData { + + private final String user; + private final String jobId; + private final String execId; + private final LinkedBlockingDeque resultContent = new LinkedBlockingDeque<>(); + private String extraMessage; + private JobStatus jobStatus = null; + private String resultLocation; + private String[] resultSetPaths = null; // remote paths for job result set + private Boolean hasNextResultPage; + private Integer errCode = null; + private String errDesc = null; + private boolean hasResult = true; + + private volatile Boolean resultFin = false; + + public ResultData(String user, String jobId, String execId) { + this.user = user; + this.jobId = jobId; + this.execId = execId; + } + + public String getJobID() { + return jobId; + } + + public String getUser() { + return user; + } + + public final String getExecID() { + return execId; + } + + public final String getResultLocation() { + return resultLocation; + } + + public final void setResultLocation(String resultLocation) { + this.resultLocation = resultLocation; + } + + public String[] getResultSetPaths() { + return resultSetPaths; + } + + public final void setResultSetPaths(String[] resultSetPaths) { + this.resultSetPaths = resultSetPaths; + } + + public Integer getErrCode() { + return errCode; + } + + public void setErrCode(Integer errCode) { + this.errCode = errCode; + } + + public String getErrDesc() { + return errDesc; + } + + public void setErrDesc(String errDesc) { + this.errDesc = errDesc; + } + + public List consumeResultContent() { + List ret = new LinkedList<>(); + resultContent.drainTo(ret, resultContent.size()); + return ret; + } + + public void appendResultContent(ResultSet resultContent) { + this.resultContent.add(resultContent); + } + + public Boolean hasNextResultPage() { + return hasNextResultPage; + } + + public void setHasNextResultPage(Boolean hasNextResultPage) { + this.hasNextResultPage = hasNextResultPage; + } + + public void setResultFin() { + this.resultFin = true; + } + + public boolean isResultFin() { + return this.resultFin; + } + + public boolean hasResult() { + return hasResult; + } + + public void setHasResult(boolean hasResult) { + this.hasResult = hasResult; + } + + public JobStatus getJobStatus() { + return jobStatus; + } + + public void setJobStatus(JobStatus jobStatus) { + this.jobStatus = jobStatus; + } + + public String getExtraMessage() { + return extraMessage; + } + + public void setExtraMessage(String extraMessage) { + this.extraMessage = extraMessage; + } + + public void updateByOperResult(LinkisOperResultAdapter adapter) { + if (adapter.getResultLocation() != null) { + setResultLocation(adapter.getResultLocation()); + } + if (adapter.getResultSetPaths() != null) { + setResultSetPaths(adapter.getResultSetPaths()); + } + if (adapter.getErrCode() != null) { + setErrCode(adapter.getErrCode()); + } + if (adapter.getErrDesc() != null) { + setErrDesc(adapter.getErrDesc()); + } + if (adapter.getResultContent() != null && adapter.resultHasNextPage() != null) { + setHasNextResultPage(adapter.resultHasNextPage()); + appendResultContent(adapter.getResultContent()); + } + if (adapter.getJobStatus() != null) { + setJobStatus(adapter.getJobStatus()); + } + } +} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/common/ResultRetriever.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/common/ResultRetriever.java new file mode 100644 index 0000000000..9997dd1f41 --- /dev/null +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/common/ResultRetriever.java @@ -0,0 +1,186 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.cli.application.interactor.job.common; + +import org.apache.linkis.cli.application.constants.CliConstants; +import org.apache.linkis.cli.application.exception.LinkisClientExecutionException; +import org.apache.linkis.cli.application.exception.LinkisClientRuntimeException; +import org.apache.linkis.cli.application.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.exception.error.ErrorLevel; +import org.apache.linkis.cli.application.observer.event.FetchResultEvent; +import org.apache.linkis.cli.application.observer.event.LinkisClientEvent; +import org.apache.linkis.cli.application.observer.listener.LinkisClientListener; +import org.apache.linkis.cli.application.operator.ujes.LinkisJobOper; +import org.apache.linkis.cli.application.utils.CliUtils; +import org.apache.linkis.cli.application.utils.LoggerManager; +import org.apache.linkis.cli.application.utils.SchedulerManager; + +import org.apache.commons.lang3.StringUtils; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ResultRetriever { + private static final Logger logger = LoggerFactory.getLogger(ResultRetriever.class); + + private LinkisJobOper linkisJobOperator; + private ResultData resultData; + + private LinkisClientListener resultListener; + private LinkisClientEvent fetchResultEvent = new FetchResultEvent(); + + public ResultRetriever( + String user, + String jobId, + String execId, + LinkisJobOper linkisJobOperator, + LinkisClientListener resultListener) { + this.linkisJobOperator = linkisJobOperator; + this.resultListener = resultListener; + this.resultData = new ResultData(user, jobId, execId); + registerResultListener(resultListener); + } + + public void retrieveResultSync() { + if (resultData.getUser() == null || resultData.getJobID() == null) { + throw new LinkisClientExecutionException( + "EXE0036", ErrorLevel.ERROR, CommonErrMsg.ExecutionErr, "user or jobID is null"); + } + resultData.updateByOperResult( + linkisJobOperator.queryJobInfo(resultData.getUser(), resultData.getJobID())); + if (resultData.getJobStatus() == null) { + throw new LinkisClientExecutionException( + "EXE0038", ErrorLevel.ERROR, CommonErrMsg.ExecutionErr, "jobStatus is null"); + } + if (!resultData.getJobStatus().isJobSuccess()) { + LoggerManager.getInformationLogger() + .info( + "Job status is not success but \'" + + resultData.getJobStatus() + + "\'. Will not try to retrieve any Result"); + resultData.setResultFin(); // inform listener to stop + return; + } + if (StringUtils.isBlank(resultData.getResultLocation())) { + throw new LinkisClientExecutionException( + "EXE0037", + ErrorLevel.WARN, + CommonErrMsg.ExecutionErr, + "Got blank ResultLocation from server. Job may not have result-set. Will not try to retrieve any Result"); + } + resultData.updateByOperResult( + linkisJobOperator.queryResultSetPaths( + resultData.getUser(), resultData.getJobID(), resultData.getResultLocation())); + + if (resultData.getResultSetPaths() == null || resultData.getResultSetPaths().length == 0) { + String msg = "Your job got no result."; + logger.warn(msg); + resultData.setResultFin(); // inform listener to stop + resultData.setHasResult(false); + return; + } + + try { + resultData.setHasResult(true); + // Thread resultConsumer = new Thread(() -> notifyResultListener()); + Thread resultThread = new Thread(() -> queryResultLoop(resultData), "Result-Retrieve-Thread"); + // SchedulerUtils.getCachedThreadPoolExecutor().execute(resultConsumer); + SchedulerManager.getCachedThreadPoolExecutor().execute(resultThread); + notifyResultListener(); + } catch (Exception e) { + logger.error("Failed to retrieve result", e); + throw e; + } + } + + public void queryResultLoop(ResultData data) { + boolean hasNext = true; + int retryCnt = 0; + final int MAX_RETRY = 30; // continues fails for 250s, then exit + int idx = 0; + try { + while (hasNext) { + try { + hasNext = queryOneResult(data, idx); + } catch (LinkisClientRuntimeException e) { + logger.error("Cannot get result:", e); + retryCnt++; + if (retryCnt >= MAX_RETRY) { + logger.error( + "Continuously failing to query result for " + + MAX_RETRY * (MAX_RETRY + 2) * 500 / 1000 + + "s. Will no longer try to query result", + e); + return; + } else { + hasNext = true; + } + CliUtils.doSleepQuietly(500l + 500l * retryCnt); // maybe server problem. sleep longer + continue; + } + idx++; + } + } catch (Exception e) { + logger.error("Something goes wrong. Job Result may be incomplete", e); + throw e; + } finally { + data.setResultFin(); + } + } + + private boolean queryOneResult(ResultData data, int idxResultSet) { + Integer curPage = 1; + boolean hasNextResult = true; + boolean hasNextPage = true; + while (hasNextPage) { + data.updateByOperResult( + linkisJobOperator.queryResultSetGivenResultSetPath( + data.getResultSetPaths(), + idxResultSet, + data.getUser(), + curPage, + CliConstants.RESULTSET_PAGE_SIZE)); + if (data.hasNextResultPage() == null) { + throw new LinkisClientExecutionException( + "EXE0040", + ErrorLevel.ERROR, + CommonErrMsg.ExecutionResultErr, + "Something foes wrong. Got null as \'hasNextPage\'."); + } + hasNextPage = data.hasNextResultPage(); + + curPage++; + hasNextResult = idxResultSet + 1 < data.getResultSetPaths().length; + } + return hasNextResult; + } + + public void registerResultListener(LinkisClientListener observer) { + this.fetchResultEvent.register(observer); + } + + public void notifyResultListener() { + if (this.fetchResultEvent.isRegistered()) { + fetchResultEvent.notifyObserver(fetchResultEvent, this.resultData); + } + } + + public void setResultFin() { + this.resultData.setResultFin(); + } +} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/data/LinkisResultSet.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/common/ResultSet.java similarity index 88% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/data/LinkisResultSet.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/common/ResultSet.java index 4fa1dcfe02..5821e14a55 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/data/LinkisResultSet.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/common/ResultSet.java @@ -15,18 +15,18 @@ * limitations under the License. */ -package org.apache.linkis.cli.application.interactor.job.data; +package org.apache.linkis.cli.application.interactor.job.common; import java.util.LinkedHashMap; import java.util.LinkedList; import java.util.List; -public class LinkisResultSet implements Cloneable { +public class ResultSet implements Cloneable { private int resultsetIdx; private List> resultMeta; private List> content; - public LinkisResultSet() {} + public ResultSet() {} public int getResultsetIdx() { return resultsetIdx; @@ -53,8 +53,8 @@ public void setContent(List> content) { } @Override - protected LinkisResultSet clone() throws CloneNotSupportedException { - LinkisResultSet ret = new LinkisResultSet(); + protected ResultSet clone() throws CloneNotSupportedException { + ResultSet ret = new ResultSet(); if (this.resultMeta != null) { List> resultMeta = null; ret.resultMeta = new LinkedList<>(); diff --git a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/help/HelpJob.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/help/HelpJob.java new file mode 100644 index 0000000000..943c54ed63 --- /dev/null +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/help/HelpJob.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.cli.application.interactor.job.help; + +import org.apache.linkis.cli.application.entity.context.CliCtx; +import org.apache.linkis.cli.application.entity.job.Job; +import org.apache.linkis.cli.application.entity.job.JobResult; +import org.apache.linkis.cli.application.present.HelpPresenter; +import org.apache.linkis.cli.application.present.model.HelpInfoModel; + +import java.util.HashMap; +import java.util.Map; + +public class HelpJob implements Job { + private CliCtx ctx; + + @Override + public void build(CliCtx ctx) { + this.ctx = ctx; + } + + @Override + public JobResult run() { + HelpInfoModel model = new HelpInfoModel(); + model.buildModel(ctx.getTemplate()); + new HelpPresenter().present(model); + return new JobResult() { + @Override + public Boolean isSuccess() { + return true; + } + + @Override + public String getMessage() { + return ""; + } + + @Override + public Map getExtraMessage() { + return new HashMap<>(); + } + }; + } + + @Override + public void onDestroy() {} +} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/interactive/InteractiveJob.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/interactive/InteractiveJob.java new file mode 100644 index 0000000000..1c04a06b5a --- /dev/null +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/interactive/InteractiveJob.java @@ -0,0 +1,267 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.cli.application.interactor.job.interactive; + +import org.apache.linkis.cli.application.constants.CliConstants; +import org.apache.linkis.cli.application.constants.CliKeys; +import org.apache.linkis.cli.application.entity.context.CliCtx; +import org.apache.linkis.cli.application.entity.job.Job; +import org.apache.linkis.cli.application.entity.job.JobResult; +import org.apache.linkis.cli.application.exception.LinkisClientExecutionException; +import org.apache.linkis.cli.application.exception.LinkisClientRuntimeException; +import org.apache.linkis.cli.application.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.exception.error.ErrorLevel; +import org.apache.linkis.cli.application.interactor.job.common.JobKiller; +import org.apache.linkis.cli.application.interactor.job.common.LogRetriever; +import org.apache.linkis.cli.application.interactor.job.common.ResultRetriever; +import org.apache.linkis.cli.application.operator.OperManager; +import org.apache.linkis.cli.application.operator.ujes.LinkisJobOper; +import org.apache.linkis.cli.application.operator.ujes.LinkisOperResultAdapter; +import org.apache.linkis.cli.application.present.LogPresenter; +import org.apache.linkis.cli.application.present.ResultPresenter; +import org.apache.linkis.cli.application.utils.CliUtils; +import org.apache.linkis.cli.application.utils.LoggerManager; + +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.exception.ExceptionUtils; + +import java.util.HashMap; +import java.util.Map; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class InteractiveJob implements Job { + + private static final Logger logger = LoggerFactory.getLogger(InteractiveJob.class); + + private CliCtx ctx; + + private Boolean isAsync = false; + + private LinkisJobOper oper; + + private InteractiveJobDesc desc; + + private String username; + + private String jobId; + + @Override + public void build(CliCtx ctx) { + this.ctx = ctx; + this.isAsync = + ctx.getVarAccess().getVarOrDefault(Boolean.class, CliKeys.LINKIS_CLIENT_ASYNC_OPT, false); + this.desc = InteractiveJobDescBuilder.build(ctx); + this.oper = (LinkisJobOper) OperManager.getNew(CliKeys.Linkis_OPER, ctx); + } + + @Override + public JobResult run() { + + // Indicator + StringBuilder infoBuilder = new StringBuilder(); + infoBuilder.append("connecting to linkis gateway:").append(oper.getServerUrl()); + LoggerManager.getInformationLogger().info(infoBuilder.toString()); + infoBuilder.setLength(0); + + // Submit + LinkisOperResultAdapter submitResult = oper.submit(desc); + CliUtils.doSleepQuietly(CliConstants.JOB_QUERY_SLEEP_MILLS); + + // JobInfo + LinkisOperResultAdapter jobInfoResult = + oper.queryJobInfo(submitResult.getUser(), submitResult.getJobID()); + oper.queryJobStatus( + jobInfoResult.getUser(), jobInfoResult.getJobID(), jobInfoResult.getStrongerExecId()); + infoBuilder.setLength(0); + infoBuilder + .append("JobId:") + .append(jobInfoResult.getJobID()) + .append(System.lineSeparator()) + .append("TaskId:") + .append(jobInfoResult.getJobID()) + .append(System.lineSeparator()) + .append("ExecId:") + .append(jobInfoResult.getStrongerExecId()); + LoggerManager.getPlaintTextLogger().info(infoBuilder.toString()); + infoBuilder.setLength(0); + + // Submit success or not + if (!jobInfoResult.getJobStatus().isJobSubmitted()) { + return new InteractiveJobResult(false, "Failed to submit job", new HashMap<>()); + } else { + // Output that job is submitted + infoBuilder.append("Job is successfully submitted!").append(System.lineSeparator()); + LoggerManager.getInformationLogger().info(infoBuilder.toString()); + infoBuilder.setLength(0); + username = submitResult.getUser(); + jobId = submitResult.getJobID(); + } + + // async job, return + if (isAsync) { + return new InteractiveJobResult( + jobInfoResult.getJobStatus().isJobSubmitted(), + "Async Submission Success", + new HashMap<>()); + } + + CliUtils.doSleepQuietly(2000l); + + // get log while running + LogRetriever logRetriever = + new LogRetriever( + jobInfoResult.getUser(), + jobInfoResult.getJobID(), + jobInfoResult.getStrongerExecId(), + true, + oper, + new LogPresenter()); + // async because we need to query job status + logRetriever.retrieveLogAsync(); + + // wait complete + jobInfoResult = waitJobComplete(submitResult.getUser(), submitResult.getJobID()); + logRetriever.waitIncLogComplete(); + + // get result-set + String outputPath = + ctx.getVarAccess().getVar(String.class, CliKeys.LINKIS_CLIENT_COMMON_OUTPUT_PATH); + ResultPresenter presenter; + if (StringUtils.isBlank(outputPath)) { + presenter = new ResultPresenter(); + } else { + presenter = new ResultPresenter(true, outputPath); + } + + ResultRetriever resultRetriever = + new ResultRetriever( + jobInfoResult.getUser(), + jobInfoResult.getJobID(), + jobInfoResult.getStrongerExecId(), + oper, + presenter); + + JobResult result = getResult(jobInfoResult, resultRetriever); + + return result; + } + + private JobResult getResult( + LinkisOperResultAdapter jobInfoResult, ResultRetriever resultRetriever) + throws LinkisClientRuntimeException { + if (!jobInfoResult.getJobStatus().isJobSuccess()) { + LoggerManager.getInformationLogger() + .info( + "Job status is not success but \'" + + jobInfoResult.getJobStatus() + + "\'. Will not try to retrieve any Result"); + Map extraMap = new HashMap<>(); + if (jobInfoResult.getErrCode() != null) { + extraMap.put("errorCode", String.valueOf(jobInfoResult.getErrCode())); + } + if (StringUtils.isNotBlank(jobInfoResult.getErrDesc())) { + extraMap.put("errorDesc", jobInfoResult.getErrDesc()); + } + return new InteractiveJobResult(false, "Execute Error!!!", extraMap); + } + InteractiveJobResult result = + new InteractiveJobResult(true, "Execute Success!!!", new HashMap<>()); + try { + resultRetriever.retrieveResultSync(); + result.setSuccess(true); + result.setMessage("execute success!!!"); + } catch (LinkisClientExecutionException e) { + if (e.getCode().equals("EXE0037")) { + result.setSuccess(true); + result.setMessage("execute success!!!"); + LoggerManager.getInformationLogger().warn(e.getMessage()); + } else { + result.setSuccess(false); + result.setMessage("execute failed!!!\n" + ExceptionUtils.getStackTrace(e)); + } + resultRetriever.setResultFin(); // inform listener to stop + } catch (Exception e) { + result.setSuccess(false); + result.setMessage("execute failed!!!\n" + ExceptionUtils.getStackTrace(e)); + resultRetriever.setResultFin(); // inform listener to stop + } + return result; + } + + private LinkisOperResultAdapter waitJobComplete(String user, String jobId) + throws LinkisClientRuntimeException { + int retryCnt = 0; + final int MAX_RETRY = 30; + + LinkisOperResultAdapter jobInfoResult = oper.queryJobInfo(user, jobId); + oper.queryJobStatus(user, jobId, jobInfoResult.getStrongerExecId()); + + while (!jobInfoResult.getJobStatus().isJobFinishedState()) { + // query progress + try { + jobInfoResult = oper.queryJobInfo(user, jobId); + oper.queryJobStatus( + jobInfoResult.getUser(), jobInfoResult.getJobID(), jobInfoResult.getStrongerExecId()); + } catch (Exception e) { + logger.warn("", e); + retryCnt++; + if (retryCnt >= MAX_RETRY) { + throw new LinkisClientExecutionException( + "EXE0013", + ErrorLevel.ERROR, + CommonErrMsg.ExecutionErr, + "Cannot get jobStatus from server continuously for {0} seconds. Client aborted! Error message: \n", + MAX_RETRY * 5 * CliConstants.JOB_QUERY_SLEEP_MILLS, + e); + } + CliUtils.doSleepQuietly( + 5 * CliConstants.JOB_QUERY_SLEEP_MILLS); // maybe server problem. sleep + // longer + continue; + } + retryCnt = 0; // reset counter + if (jobInfoResult.getJobStatus().isJobAbnormalStatus()) { + throw new LinkisClientExecutionException( + "EXE0006", + ErrorLevel.ERROR, + CommonErrMsg.ExecutionErr, + "Job is in abnormal status: " + CliUtils.GSON.toJson(jobInfoResult)); + } + CliUtils.doSleepQuietly(CliConstants.JOB_QUERY_SLEEP_MILLS); + } + return jobInfoResult; + } + + @Override + public void onDestroy() { + if (StringUtils.isBlank(username) || StringUtils.isBlank(jobId)) { + logger.warn("Failed to kill job username or jobId is blank"); + return; + } + if (isAsync) { + return; + } + try { + new JobKiller(oper).doKill(username, jobId); + } catch (Exception e) { + logger.error("Failed to kill job", e); + } + } +} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/desc/LinkisSubmitDesc.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/interactive/InteractiveJobDesc.java similarity index 93% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/desc/LinkisSubmitDesc.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/interactive/InteractiveJobDesc.java index 49cdd1a547..e594d9cc23 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/desc/LinkisSubmitDesc.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/interactive/InteractiveJobDesc.java @@ -15,13 +15,11 @@ * limitations under the License. */ -package org.apache.linkis.cli.application.interactor.job.desc; - -import org.apache.linkis.cli.common.entity.job.JobDescription; +package org.apache.linkis.cli.application.interactor.job.interactive; import java.util.Map; -public class LinkisSubmitDesc implements JobDescription { +public class InteractiveJobDesc { private String submitUser; private String proxyUser; private String creator; diff --git a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/interactive/InteractiveJobDescBuilder.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/interactive/InteractiveJobDescBuilder.java new file mode 100644 index 0000000000..2b0b20188a --- /dev/null +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/interactive/InteractiveJobDescBuilder.java @@ -0,0 +1,177 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.cli.application.interactor.job.interactive; + +import org.apache.linkis.cli.application.constants.CliConstants; +import org.apache.linkis.cli.application.constants.CliKeys; +import org.apache.linkis.cli.application.constants.LinkisKeys; +import org.apache.linkis.cli.application.entity.context.CliCtx; +import org.apache.linkis.cli.application.entity.var.VarAccess; +import org.apache.linkis.cli.application.interactor.job.common.KeyParser; +import org.apache.linkis.cli.application.operator.ujes.LinkisJobOper; +import org.apache.linkis.cli.application.operator.ujes.UJESClientFactory; +import org.apache.linkis.cli.application.utils.CliUtils; +import org.apache.linkis.cli.application.utils.LoggerManager; + +import org.apache.commons.lang3.StringUtils; + +import java.util.*; + +public class InteractiveJobDescBuilder { + + public static InteractiveJobDesc build(CliCtx ctx) { + InteractiveJobDesc desc = new InteractiveJobDesc(); + + VarAccess stdVarAccess = ctx.getVarAccess(); + + Map confMap = stdVarAccess.getVar(Map.class, CliKeys.JOB_PARAM_CONF); + Map runtimeMap = stdVarAccess.getVar(Map.class, CliKeys.JOB_PARAM_RUNTIME); + Map varMap = stdVarAccess.getVar(Map.class, CliKeys.JOB_PARAM_VAR); + Map labelMap = stdVarAccess.getVar(Map.class, CliKeys.JOB_LABEL); + Map sourceMap = stdVarAccess.getVar(Map.class, CliKeys.JOB_SOURCE); + Map executionMap = stdVarAccess.getVar(Map.class, CliKeys.JOB_EXEC); + + confMap = confMap == null ? new HashMap<>() : confMap; + runtimeMap = runtimeMap == null ? new HashMap<>() : runtimeMap; + varMap = varMap == null ? new HashMap<>() : varMap; + labelMap = labelMap == null ? new HashMap<>() : labelMap; + sourceMap = sourceMap == null ? new HashMap<>() : sourceMap; + executionMap = executionMap == null ? new HashMap<>() : executionMap; + + /** remove key prefix of all keys in map type params. e.g. kv in confMap, labelMap etc. */ + confMap = KeyParser.removePrefixForKeysInMap(confMap); + runtimeMap = KeyParser.removePrefixForKeysInMap(runtimeMap); + labelMap = KeyParser.removePrefixForKeysInMap(labelMap); + sourceMap = KeyParser.removePrefixForKeysInMap(sourceMap); + executionMap = KeyParser.removePrefixForKeysInMap(executionMap); + + /** remove key prefix of non-map type params */ + for (String key : stdVarAccess.getAllVarKeys()) { + Object val = stdVarAccess.getVar(Object.class, key); + if (!(val instanceof Map) && val != null) { + // note that we allow it to overwrite existing values in map + if (StringUtils.startsWithIgnoreCase(key, CliKeys.JOB_PARAM_CONF)) { + KeyParser.removePrefixAndPutValToMap(confMap, key, val, CliKeys.JOB_PARAM_CONF); + } else if (StringUtils.startsWithIgnoreCase(key, CliKeys.JOB_PARAM_VAR)) { + KeyParser.removePrefixAndPutValToMap(varMap, key, val, CliKeys.JOB_PARAM_VAR); + } else if (StringUtils.startsWithIgnoreCase(key, CliKeys.JOB_PARAM_RUNTIME)) { + KeyParser.removePrefixAndPutValToMap(runtimeMap, key, val, CliKeys.JOB_PARAM_RUNTIME); + } else if (StringUtils.startsWithIgnoreCase(key, CliKeys.JOB_EXEC)) { + KeyParser.removePrefixAndPutValToMap(executionMap, key, val, CliKeys.JOB_EXEC); + } else if (StringUtils.startsWithIgnoreCase(key, CliKeys.JOB_LABEL)) { + KeyParser.removePrefixAndPutValToMap(labelMap, key, val, CliKeys.JOB_LABEL); + } else if (StringUtils.startsWithIgnoreCase(key, CliKeys.JOB_SOURCE)) { + KeyParser.removePrefixAndPutValToMap(sourceMap, key, val, CliKeys.JOB_SOURCE); + } else if (StringUtils.startsWithIgnoreCase(key, CliKeys.LINKIS_CLIENT_COMMON)) { + // do nothing + } else { + // confMap.put(key, stdVarAccess.getVar(Object.class, key)); + } + } + } + + Boolean asyncSubmission = + stdVarAccess.getVarOrDefault(Boolean.class, CliKeys.LINKIS_CLIENT_ASYNC_OPT, false); + + String creator; + if (!asyncSubmission) { + creator = + stdVarAccess.getVarOrDefault( + String.class, CliKeys.JOB_COMMON_CREATOR, CliConstants.JOB_CREATOR_DEFAULT); + } else { + creator = + stdVarAccess.getVarOrDefault( + String.class, CliKeys.JOB_COMMON_CREATOR, CliConstants.JOB_CREATOR_ASYNC_DEFAULT); + } + String code = stdVarAccess.getVar(String.class, CliKeys.JOB_EXEC_CODE); + String engineType = stdVarAccess.getVar(String.class, CliKeys.JOB_LABEL_ENGINE_TYPE); + String runType = stdVarAccess.getVar(String.class, CliKeys.JOB_LABEL_CODE_TYPE); + String scriptPath = + stdVarAccess.getVarOrDefault(String.class, CliKeys.JOB_SOURCE_SCRIPT_PATH, "LinkisCli"); + + String osUser = System.getProperty(CliKeys.LINUX_USER_KEY); + String[] adminUsers = StringUtils.split(CliKeys.ADMIN_USERS, ','); + Set adminSet = new HashSet<>(); + for (String admin : adminUsers) { + adminSet.add(admin); + } + String submitUsr = CliUtils.getSubmitUser(stdVarAccess, osUser, adminSet); + String proxyUsr = CliUtils.getProxyUser(stdVarAccess, submitUsr, adminSet); + + String enableExecuteOnce = + stdVarAccess.getVarOrDefault(String.class, CliKeys.JOB_LABEL_EXECUTEONCE, "true"); + // default executeOnce-mode + if (Boolean.parseBoolean(enableExecuteOnce)) { + labelMap.put(LinkisKeys.KEY_EXECUTEONCE, ""); + } else { + labelMap.remove(LinkisKeys.KEY_EXECUTEONCE); + } + String codePath = stdVarAccess.getVar(String.class, CliKeys.JOB_COMMON_CODE_PATH); + Object extraArgsObj = stdVarAccess.getVar(Object.class, CliKeys.JOB_EXTRA_ARGUMENTS); + if (extraArgsObj != null + && extraArgsObj instanceof String[] + && StringUtils.isBlank(code) + && StringUtils.isBlank(codePath)) { + String[] extraArgs = (String[]) extraArgsObj; + codePath = extraArgs[0]; + if (extraArgs.length > 1) { + runtimeMap.put( + LinkisKeys.EXTRA_ARGUMENTS, Arrays.copyOfRange(extraArgs, 1, extraArgs.length)); + } + } + + if (StringUtils.isBlank(code) && StringUtils.isNotBlank(codePath)) { + try { + code = CliUtils.readFile(codePath); + } catch (Exception e) { + LoggerManager.getInformationLogger().error("Failed to read file", e); + throw e; + } + } + + executionMap.put(LinkisKeys.KEY_CODE, code); + labelMap.put(LinkisKeys.KEY_ENGINETYPE, engineType); + labelMap.put(LinkisKeys.KEY_CODETYPE, runType); + labelMap.put(LinkisKeys.KEY_USER_CREATOR, proxyUsr + "-" + creator); + sourceMap.put(LinkisKeys.KEY_SCRIPT_PATH, scriptPath); + if (ctx.getExtraMap().containsKey(CliKeys.VERSION)) { + sourceMap.put(LinkisKeys.CLI_VERSION, ctx.getExtraMap().get(CliKeys.VERSION)); + } + runtimeMap.put(LinkisKeys.KEY_HIVE_RESULT_DISPLAY_TBALE, true); + + desc.setCreator(creator); + desc.setParamConfMap(confMap); + desc.setParamRunTimeMap(runtimeMap); + desc.setParamVarsMap(varMap); + desc.setLabelMap(labelMap); + desc.setSourceMap(sourceMap); + desc.setExecutionMap(executionMap); + desc.setSubmitUser(submitUsr); + desc.setProxyUser(proxyUsr); + + return desc; + } + + public static LinkisJobOper generateOperator(CliCtx ctx) { + LinkisJobOper linkisJobOperator = new LinkisJobOper(); + linkisJobOperator.setUJESClient(UJESClientFactory.getReusable(ctx.getVarAccess())); + linkisJobOperator.setServerUrl( + ctx.getVarAccess().getVar(String.class, CliKeys.LINKIS_COMMON_GATEWAY_URL)); + return linkisJobOperator; + } +} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/interactive/InteractiveJobResult.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/interactive/InteractiveJobResult.java new file mode 100644 index 0000000000..92999169ed --- /dev/null +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/interactive/InteractiveJobResult.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.cli.application.interactor.job.interactive; + +import org.apache.linkis.cli.application.entity.job.JobResult; + +import java.util.Map; + +public class InteractiveJobResult implements JobResult { + private Boolean success; + private String message; + private Map extraMessage; + + public InteractiveJobResult(Boolean success, String message, Map extraMessage) { + this.success = success; + this.message = message; + this.extraMessage = extraMessage; + } + + @Override + public Boolean isSuccess() { + return success; + } + + @Override + public String getMessage() { + return message; + } + + public void setMessage(String message) { + this.message = message; + } + + @Override + public Map getExtraMessage() { + return extraMessage; + } + + public void setExtraMessage(Map extraMessage) { + this.extraMessage = extraMessage; + } + + public void setSuccess(Boolean success) { + this.success = success; + } +} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/desc/LinkisJobManDesc.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/jobcmd/JobCmdDesc.java similarity index 82% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/desc/LinkisJobManDesc.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/jobcmd/JobCmdDesc.java index da4f5d1c47..99f0365004 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/desc/LinkisJobManDesc.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/jobcmd/JobCmdDesc.java @@ -15,18 +15,25 @@ * limitations under the License. */ -package org.apache.linkis.cli.application.interactor.job.desc; - -import org.apache.linkis.cli.common.entity.job.JobDescription; +package org.apache.linkis.cli.application.interactor.job.jobcmd; import java.util.Map; -public class LinkisJobManDesc implements JobDescription { +public class JobCmdDesc { private String jobId; private String user; + private JobCmdSubType subType; private Map params; + public JobCmdSubType getSubType() { + return subType; + } + + public void setSubType(JobCmdSubType subType) { + this.subType = subType; + } + public String getJobID() { return jobId; } diff --git a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/jobcmd/JobCmdDescBuilder.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/jobcmd/JobCmdDescBuilder.java new file mode 100644 index 0000000000..1e52e6e532 --- /dev/null +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/jobcmd/JobCmdDescBuilder.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.cli.application.interactor.job.jobcmd; + +import org.apache.linkis.cli.application.constants.CliKeys; +import org.apache.linkis.cli.application.entity.context.CliCtx; +import org.apache.linkis.cli.application.utils.CliUtils; + +import org.apache.commons.lang3.StringUtils; + +import java.util.HashSet; +import java.util.Set; + +public class JobCmdDescBuilder { + public static JobCmdDesc build(CliCtx ctx) { + JobCmdDesc desc = new JobCmdDesc(); + String osUser = System.getProperty(CliKeys.LINUX_USER_KEY); + String[] adminUsers = StringUtils.split(CliKeys.ADMIN_USERS, ','); + Set adminSet = new HashSet<>(); + for (String admin : adminUsers) { + adminSet.add(admin); + } + String submitUsr = CliUtils.getSubmitUser(ctx.getVarAccess(), osUser, adminSet); + + JobCmdSubType subType = null; + + String jobId = null; + if (ctx.getVarAccess().hasVar(CliKeys.LINKIS_CLIENT_KILL_OPT)) { + jobId = ctx.getVarAccess().getVar(String.class, CliKeys.LINKIS_CLIENT_KILL_OPT); + subType = JobCmdSubType.KILL; + } else if (ctx.getVarAccess().hasVar(CliKeys.LINKIS_CLIENT_STATUS_OPT)) { + jobId = ctx.getVarAccess().getVar(String.class, CliKeys.LINKIS_CLIENT_STATUS_OPT); + subType = JobCmdSubType.STATUS; + } else if (ctx.getVarAccess().hasVar(CliKeys.LINKIS_CLIENT_DESC_OPT)) { + jobId = ctx.getVarAccess().getVar(String.class, CliKeys.LINKIS_CLIENT_DESC_OPT); + subType = JobCmdSubType.DESC; + } else if (ctx.getVarAccess().hasVar(CliKeys.LINKIS_CLIENT_LOG_OPT)) { + jobId = ctx.getVarAccess().getVar(String.class, CliKeys.LINKIS_CLIENT_LOG_OPT); + subType = JobCmdSubType.LOG; + } else if (ctx.getVarAccess().hasVar(CliKeys.LINKIS_CLIENT_RESULT_OPT)) { + jobId = ctx.getVarAccess().getVar(String.class, CliKeys.LINKIS_CLIENT_RESULT_OPT); + subType = JobCmdSubType.RESULT; + } else if (ctx.getVarAccess().hasVar(CliKeys.LINKIS_CLIENT_LIST_OPT)) { + jobId = ctx.getVarAccess().getVar(String.class, CliKeys.LINKIS_CLIENT_LIST_OPT); + subType = JobCmdSubType.LIST; + } + desc.setSubType(subType); + desc.setJobId(jobId); + desc.setUser(submitUsr); + return desc; + } +} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/jobcmd/JobCmdJob.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/jobcmd/JobCmdJob.java new file mode 100644 index 0000000000..e39be38f42 --- /dev/null +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/jobcmd/JobCmdJob.java @@ -0,0 +1,212 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.cli.application.interactor.job.jobcmd; + +import org.apache.linkis.cli.application.constants.CliKeys; +import org.apache.linkis.cli.application.entity.context.CliCtx; +import org.apache.linkis.cli.application.entity.job.Job; +import org.apache.linkis.cli.application.entity.job.JobResult; +import org.apache.linkis.cli.application.exception.LinkisClientExecutionException; +import org.apache.linkis.cli.application.exception.LinkisClientRuntimeException; +import org.apache.linkis.cli.application.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.exception.error.ErrorLevel; +import org.apache.linkis.cli.application.interactor.job.common.JobKiller; +import org.apache.linkis.cli.application.interactor.job.common.KillResult; +import org.apache.linkis.cli.application.interactor.job.common.LogRetriever; +import org.apache.linkis.cli.application.interactor.job.common.ResultRetriever; +import org.apache.linkis.cli.application.operator.OperManager; +import org.apache.linkis.cli.application.operator.ujes.LinkisJobOper; +import org.apache.linkis.cli.application.operator.ujes.LinkisOperResultAdapter; +import org.apache.linkis.cli.application.present.JobInfoPresenter; +import org.apache.linkis.cli.application.present.LogPresenter; +import org.apache.linkis.cli.application.present.ResultPresenter; +import org.apache.linkis.cli.application.present.model.LinkisJobInfoModel; +import org.apache.linkis.cli.application.utils.LoggerManager; + +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.exception.ExceptionUtils; + +import java.util.HashMap; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class JobCmdJob implements Job { + private static final Logger logger = LoggerFactory.getLogger(JobCmdJob.class); + + protected CliCtx ctx; + + protected LinkisJobOper oper; + + protected JobCmdDesc desc; + + @Override + public void build(CliCtx ctx) { + this.ctx = ctx; + this.desc = JobCmdDescBuilder.build(ctx); + this.oper = (LinkisJobOper) OperManager.getNew(CliKeys.Linkis_OPER, ctx); + } + + @Override + public JobResult run() { + JobCmdSubType subType = desc.getSubType(); + if (!(subType instanceof JobCmdSubType)) { + throw new LinkisClientExecutionException( + "EXE0030", + ErrorLevel.ERROR, + CommonErrMsg.ExecutionErr, + "JobSubType is not instance of JobManSubType"); + } + JobCmdJobResult result = new JobCmdJobResult(true, "Execute Success!!!", new HashMap<>()); + + switch (subType) { + case STATUS: + try { + LinkisOperResultAdapter jobInfoResult = + oper.queryJobInfo(desc.getUser(), desc.getJobID()); + LinkisJobInfoModel model = new LinkisJobInfoModel(); + model.buildModel(jobInfoResult); + new JobInfoPresenter().present(model); + } catch (Exception e) { + result.setSuccess(false); + result.setMessage(ExceptionUtils.getStackTrace(e)); + } + if (!result.isSuccess()) { + LoggerManager.getPlaintTextLogger() + .error("Failed to get job-info. Message: " + result.getMessage()); + } + return result; + case LOG: + try { + // get log while running + LinkisOperResultAdapter jobInfoResult = + oper.queryJobInfo(desc.getUser(), desc.getJobID()); + LogRetriever logRetriever = + new LogRetriever( + jobInfoResult.getUser(), + jobInfoResult.getJobID(), + jobInfoResult.getStrongerExecId(), + false, + oper, + new LogPresenter()); + // async because we need to query job status + logRetriever.retrieveLogAsync(); + logRetriever.waitIncLogComplete(); + } catch (Exception e) { + result.setSuccess(false); + result.setMessage(ExceptionUtils.getStackTrace(e)); + } + if (!result.isSuccess()) { + LoggerManager.getInformationLogger() + .error("Failed to get log. Message: " + result.getMessage()); + } + return result; + case RESULT: + // get log while running + LinkisOperResultAdapter jobInfoResult = oper.queryJobInfo(desc.getUser(), desc.getJobID()); + // get result-set + String outputPath = + ctx.getVarAccess().getVar(String.class, CliKeys.LINKIS_CLIENT_COMMON_OUTPUT_PATH); + ResultPresenter presenter; + if (StringUtils.isBlank(outputPath)) { + presenter = new ResultPresenter(); + } else { + presenter = new ResultPresenter(true, outputPath); + } + + ResultRetriever resultRetriever = + new ResultRetriever( + jobInfoResult.getUser(), + jobInfoResult.getJobID(), + jobInfoResult.getStrongerExecId(), + oper, + presenter); + + result = getResult(resultRetriever); + if (!result.isSuccess()) { + LoggerManager.getInformationLogger() + .error("Failed to get result. Message: " + result.getMessage()); + } + return result; + case KILL: + JobKiller jobKiller = new JobKiller(oper); + KillResult killResult; + try { + killResult = jobKiller.doKill(desc.getUser(), desc.getJobID()); + } catch (Exception e) { + killResult = + new KillResult( + false, + "Failed to kill job. Messgae: " + ExceptionUtils.getStackTrace(e), + new HashMap<>()); + } + if (killResult.isSuccess()) { + LoggerManager.getPlaintTextLogger().info("Kill Success. Current job-info:"); + } else { + LoggerManager.getPlaintTextLogger() + .error("Kill Failed. Messgae: " + killResult.getMessage() + "\n Current job-info:"); + } + try { + LinkisOperResultAdapter jobInfoResult2 = + oper.queryJobInfo(desc.getUser(), desc.getJobID()); + LinkisJobInfoModel model = new LinkisJobInfoModel(); + model.buildModel(jobInfoResult2); + new JobInfoPresenter().present(model); + } catch (Exception e) { + LoggerManager.getInformationLogger().error("Failed to get jobInfo", e); + } + return new JobCmdJobResult( + killResult.isSuccess(), killResult.getMessage(), killResult.getExtraMessage()); + // case LIST: + // break; + // case JOB_DESC: + // break; + default: + return new JobCmdJobResult( + false, "JobSubType + \"" + subType + "\" is not supported", new HashMap<>()); + } + } + + private JobCmdJobResult getResult(ResultRetriever resultRetriever) + throws LinkisClientRuntimeException { + JobCmdJobResult result = new JobCmdJobResult(true, "Execute Success!!!", new HashMap<>()); + try { + resultRetriever.retrieveResultSync(); + result.setSuccess(true); + result.setMessage("execute success!!!"); + } catch (LinkisClientExecutionException e) { + if (e.getCode().equals("EXE0037")) { + result.setSuccess(true); + result.setMessage("execute success!!!"); + LoggerManager.getInformationLogger().warn(e.getMessage()); + } else { + result.setSuccess(false); + result.setMessage("execute failed!!!\n" + ExceptionUtils.getStackTrace(e)); + } + resultRetriever.setResultFin(); // inform listener to stop + } catch (Exception e) { + result.setSuccess(false); + result.setMessage("execute failed!!!\n" + ExceptionUtils.getStackTrace(e)); + resultRetriever.setResultFin(); // inform listener to stop + } + return result; + } + + @Override + public void onDestroy() {} +} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/jobcmd/JobCmdJobResult.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/jobcmd/JobCmdJobResult.java new file mode 100644 index 0000000000..c5b9b43535 --- /dev/null +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/jobcmd/JobCmdJobResult.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.cli.application.interactor.job.jobcmd; + +import org.apache.linkis.cli.application.entity.job.JobResult; + +import java.util.Map; + +public class JobCmdJobResult implements JobResult { + private Boolean success; + private String message; + private Map extraMessage; + + public JobCmdJobResult(Boolean success, String message, Map extraMessage) { + this.success = success; + this.message = message; + this.extraMessage = extraMessage; + } + + @Override + public Boolean isSuccess() { + return success; + } + + @Override + public String getMessage() { + return message; + } + + public void setMessage(String message) { + this.message = message; + } + + @Override + public Map getExtraMessage() { + return extraMessage; + } + + public void setExtraMessage(Map extraMessage) { + this.extraMessage = extraMessage; + } + + public void setSuccess(Boolean success) { + this.success = success; + } +} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/subtype/LinkisManSubType.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/jobcmd/JobCmdSubType.java similarity index 81% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/subtype/LinkisManSubType.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/jobcmd/JobCmdSubType.java index 5df4b07976..b2d298995a 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/subtype/LinkisManSubType.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/jobcmd/JobCmdSubType.java @@ -15,11 +15,9 @@ * limitations under the License. */ -package org.apache.linkis.cli.application.interactor.job.subtype; +package org.apache.linkis.cli.application.interactor.job.jobcmd; -import org.apache.linkis.cli.common.entity.job.JobSubType; - -public enum LinkisManSubType implements JobSubType { +public enum JobCmdSubType { KILL("kill"), LOG("log"), DESC("desc"), @@ -29,11 +27,10 @@ public enum LinkisManSubType implements JobSubType { private String name; - LinkisManSubType(String name) { + JobCmdSubType(String name) { this.name = name; } - @Override public String getName() { return this.name; } diff --git a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/once/LinkisOnceJob.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/once/LinkisOnceJob.java new file mode 100644 index 0000000000..fac387998e --- /dev/null +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/once/LinkisOnceJob.java @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.cli.application.interactor.job.once; + +import org.apache.linkis.cli.application.constants.CliKeys; +import org.apache.linkis.cli.application.entity.context.CliCtx; +import org.apache.linkis.cli.application.entity.job.Job; +import org.apache.linkis.cli.application.entity.job.JobResult; +import org.apache.linkis.cli.application.entity.job.JobStatus; +import org.apache.linkis.cli.application.operator.OperManager; +import org.apache.linkis.cli.application.operator.once.OnceJobOper; +import org.apache.linkis.cli.application.utils.CliUtils; +import org.apache.linkis.cli.application.utils.LoggerManager; +import org.apache.linkis.cli.application.utils.SchedulerManager; + +import java.util.HashMap; +import java.util.concurrent.CountDownLatch; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class LinkisOnceJob implements Job { + + private static final Logger logger = LoggerFactory.getLogger(LinkisOnceJob.class); + + private Boolean isAsync = false; + private OnceJobOper oper; + + @Override + public void build(CliCtx ctx) { + this.isAsync = + ctx.getVarAccess().getVarOrDefault(Boolean.class, CliKeys.LINKIS_CLIENT_ASYNC_OPT, false); + oper = (OnceJobOper) OperManager.getNew(CliKeys.LINKIS_ONCE, ctx); + } + + @Override + public JobResult run() { + StringBuilder infoBuilder = new StringBuilder(); + infoBuilder.append("connecting to linkis gateway:").append(oper.getServerUrl()); + LoggerManager.getInformationLogger().info(infoBuilder.toString()); + + /** submit */ + oper.submit(); + JobStatus jobStatus = oper.getStatus(); + infoBuilder.setLength(0); + infoBuilder.append("JobId:").append(oper.getJobID()).append(System.lineSeparator()); + LoggerManager.getPlaintTextLogger().info(infoBuilder.toString()); + if (isAsync && jobStatus != null && jobStatus.isJobSubmitted()) { + return new OnceJobResult(true, "Submit Success!!!", new HashMap<>()); + } + + /** getLog */ + CountDownLatch latch = new CountDownLatch(1); + try { + Thread logConsumer = new Thread(() -> ProcessLog(latch), "Log-Consumer"); + SchedulerManager.getCachedThreadPoolExecutor().execute(logConsumer); + } catch (Exception e) { + logger.warn("Failed to retrieve log", e); + } + + /** wait complete */ + oper.waitForComplete(); + try { + latch.await(); + } catch (Exception e) { + // ignore + } + + JobStatus finalStatus = oper.getStatus(); + + if (finalStatus.isJobSuccess()) { + return new OnceJobResult(true, "Execute Success!!!", new HashMap<>()); + } else { + return new OnceJobResult(false, "Execute Failure!!!", new HashMap<>()); + } + } + + @Override + public void onDestroy() { + oper.kill(); + } + + private void ProcessLog(CountDownLatch latch) { + while (!oper.isLogFin()) { + String log = oper.getCurrentLog(); + LoggerManager.getPlaintTextLogger().info(log); + CliUtils.doSleepQuietly(2000l); + } + latch.countDown(); + } +} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/once/OnceJobResult.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/once/OnceJobResult.java new file mode 100644 index 0000000000..2a14f76a6f --- /dev/null +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/once/OnceJobResult.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.cli.application.interactor.job.once; + +import org.apache.linkis.cli.application.entity.job.JobResult; + +import java.util.Map; + +public class OnceJobResult implements JobResult { + private Boolean success; + private String message; + private Map extraMessage; + + public OnceJobResult(Boolean success, String message, Map extraMessage) { + this.success = success; + this.message = message; + this.extraMessage = extraMessage; + } + + @Override + public Boolean isSuccess() { + return success; + } + + @Override + public String getMessage() { + return message; + } + + public void setMessage(String message) { + this.message = message; + } + + @Override + public Map getExtraMessage() { + return extraMessage; + } + + public void setExtraMessage(Map extraMessage) { + this.extraMessage = extraMessage; + } + + public void setSuccess(Boolean success) { + this.success = success; + } +} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/version/VersionJob.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/version/VersionJob.java new file mode 100644 index 0000000000..599f97904f --- /dev/null +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/version/VersionJob.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.cli.application.interactor.job.version; + +import org.apache.linkis.cli.application.constants.CliKeys; +import org.apache.linkis.cli.application.entity.context.CliCtx; +import org.apache.linkis.cli.application.entity.job.Job; +import org.apache.linkis.cli.application.entity.job.JobResult; +import org.apache.linkis.cli.application.utils.LoggerManager; + +import java.util.HashMap; +import java.util.Map; + +public class VersionJob implements Job { + private CliCtx ctx; + + @Override + public void build(CliCtx cliCtx) { + this.ctx = cliCtx; + } + + @Override + public JobResult run() { + String version = (String) ctx.getExtraMap().get(CliKeys.VERSION); + Map extraMap = new HashMap<>(); + extraMap.put(CliKeys.VERSION, version); + LoggerManager.getPlaintTextLogger().info("Version=" + version); + return new VersionJobResult(true, "ok", extraMap); + } + + @Override + public void onDestroy() {} +} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/operator/JobOperatorBuilder.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/version/VersionJobResult.java similarity index 55% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/operator/JobOperatorBuilder.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/version/VersionJobResult.java index 8727ec495e..e2f12cd7c2 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/operator/JobOperatorBuilder.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/version/VersionJobResult.java @@ -15,20 +15,35 @@ * limitations under the License. */ -package org.apache.linkis.cli.core.operator; +package org.apache.linkis.cli.application.interactor.job.version; -import org.apache.linkis.cli.common.entity.operator.JobOperator; -import org.apache.linkis.cli.common.entity.var.VarAccess; -import org.apache.linkis.cli.core.builder.BuildableByVarAccess; +import org.apache.linkis.cli.application.entity.job.JobResult; + +import java.util.Map; + +public class VersionJobResult implements JobResult { + private Boolean success; + private String message; + private Map extraMsg; + + public VersionJobResult(Boolean success, String message, Map extraMsg) { + this.success = success; + this.message = message; + this.extraMsg = extraMsg; + } + + @Override + public Boolean isSuccess() { + return success; + } -public abstract class JobOperatorBuilder extends BuildableByVarAccess { @Override - public JobOperatorBuilder setStdVarAccess(VarAccess varAccess) { - return (JobOperatorBuilder) super.setStdVarAccess(varAccess); + public String getMessage() { + return message; } @Override - public JobOperatorBuilder setSysVarAccess(VarAccess varAccess) { - return (JobOperatorBuilder) super.setSysVarAccess(varAccess); + public Map getExtraMessage() { + return extraMsg; } } diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/properties/ClientProperties.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/properties/ClientProperties.java similarity index 95% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/properties/ClientProperties.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/properties/ClientProperties.java index 00c36a3032..6d165eeed6 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/properties/ClientProperties.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/properties/ClientProperties.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.linkis.cli.common.entity.properties; +package org.apache.linkis.cli.application.interactor.properties; import java.util.HashMap; diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/properties/StdPropsLoader.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/properties/PropertiesLoader.java similarity index 82% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/properties/StdPropsLoader.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/properties/PropertiesLoader.java index f6917206ab..d66cfe605d 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/properties/StdPropsLoader.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/properties/PropertiesLoader.java @@ -15,24 +15,22 @@ * limitations under the License. */ -package org.apache.linkis.cli.core.interactor.properties; +package org.apache.linkis.cli.application.interactor.properties; -import org.apache.linkis.cli.common.entity.properties.ClientProperties; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.exception.PropsException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; -import org.apache.linkis.cli.core.interactor.properties.reader.PropertiesReader; +import org.apache.linkis.cli.application.exception.PropsException; +import org.apache.linkis.cli.application.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.exception.error.ErrorLevel; +import org.apache.linkis.cli.application.interactor.properties.reader.PropertiesReader; import java.util.*; -public class StdPropsLoader implements PropertiesLoader { +public class PropertiesLoader { Map readersMap; - public StdPropsLoader() { + public PropertiesLoader() { this.readersMap = new HashMap<>(); } - @Override public PropertiesLoader setPropertiesReaders(PropertiesReader[] readers) { this.readersMap = new HashMap<>(); for (PropertiesReader reader : readers) { @@ -41,7 +39,6 @@ public PropertiesLoader setPropertiesReaders(PropertiesReader[] readers) { return this; } - @Override public PropertiesLoader addPropertiesReader(PropertiesReader reader) { if (reader != null) { readersMap.put(reader.getPropsId(), reader); @@ -49,7 +46,6 @@ public PropertiesLoader addPropertiesReader(PropertiesReader reader) { return this; } - @Override public PropertiesLoader addPropertiesReaders(PropertiesReader[] readers) { if (readers != null && readers.length > 0) { for (PropertiesReader reader : readers) { @@ -59,12 +55,10 @@ public PropertiesLoader addPropertiesReaders(PropertiesReader[] readers) { return this; } - @Override public void removePropertiesReader(String identifier) { readersMap.remove(identifier); } - @Override public ClientProperties[] loadProperties() { checkInit(); List propsList = new ArrayList<>(); @@ -81,7 +75,6 @@ public ClientProperties[] loadProperties() { return propsList.toArray(new ClientProperties[propsList.size()]); } - @Override public void checkInit() { if (readersMap == null || readersMap.size() == 0) { throw new PropsException( diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/properties/PropsFilesScanner.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/properties/PropsFilesScanner.java similarity index 81% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/properties/PropsFilesScanner.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/properties/PropsFilesScanner.java index 2bd120e5ca..8e343d0948 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/properties/PropsFilesScanner.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/properties/PropsFilesScanner.java @@ -15,14 +15,14 @@ * limitations under the License. */ -package org.apache.linkis.cli.core.interactor.properties; +package org.apache.linkis.cli.application.interactor.properties; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.constants.CommonConstants; -import org.apache.linkis.cli.core.exception.PropsException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; -import org.apache.linkis.cli.core.interactor.properties.reader.PropertiesReader; -import org.apache.linkis.cli.core.interactor.properties.reader.PropsFileReader; +import org.apache.linkis.cli.application.constants.CliConstants; +import org.apache.linkis.cli.application.exception.PropsException; +import org.apache.linkis.cli.application.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.exception.error.ErrorLevel; +import org.apache.linkis.cli.application.interactor.properties.reader.PropertiesReader; +import org.apache.linkis.cli.application.interactor.properties.reader.PropsFileReader; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; @@ -51,7 +51,7 @@ public List getPropsFiles(String rootPath) { try { files = (List) - FileUtils.listFiles(new File(rootPath), CommonConstants.CONFIG_EXTENSION, false); + FileUtils.listFiles(new File(rootPath), CliConstants.CONFIG_EXTENSION, false); } catch (Exception e) { throw new PropsException( "PRP0005", diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/properties/reader/PropertiesReader.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/properties/reader/PropertiesReader.java similarity index 93% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/properties/reader/PropertiesReader.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/properties/reader/PropertiesReader.java index 3c86f1b1c3..65bc2d1e06 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/properties/reader/PropertiesReader.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/properties/reader/PropertiesReader.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.linkis.cli.core.interactor.properties.reader; +package org.apache.linkis.cli.application.interactor.properties.reader; import java.util.Properties; diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/properties/reader/PropsFileReader.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/properties/reader/PropsFileReader.java similarity index 90% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/properties/reader/PropsFileReader.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/properties/reader/PropsFileReader.java index 024a83311e..7bd23da140 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/properties/reader/PropsFileReader.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/properties/reader/PropsFileReader.java @@ -15,11 +15,11 @@ * limitations under the License. */ -package org.apache.linkis.cli.core.interactor.properties.reader; +package org.apache.linkis.cli.application.interactor.properties.reader; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.exception.PropsException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.exception.PropsException; +import org.apache.linkis.cli.application.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.exception.error.ErrorLevel; import org.apache.commons.lang3.StringUtils; diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/validate/LinkisSubmitValidator.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/validate/InteractiveDescValidator.java similarity index 76% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/validate/LinkisSubmitValidator.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/validate/InteractiveDescValidator.java index fa0f31dc1e..03eef75612 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/validate/LinkisSubmitValidator.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/validate/InteractiveDescValidator.java @@ -18,67 +18,56 @@ package org.apache.linkis.cli.application.interactor.validate; import org.apache.linkis.cli.application.constants.LinkisKeys; -import org.apache.linkis.cli.application.interactor.job.LinkisSubmitJob; -import org.apache.linkis.cli.application.interactor.job.desc.LinkisSubmitDesc; -import org.apache.linkis.cli.common.entity.validate.Validator; -import org.apache.linkis.cli.common.exception.LinkisClientRuntimeException; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.exception.ValidateException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.exception.LinkisClientRuntimeException; +import org.apache.linkis.cli.application.exception.ValidateException; +import org.apache.linkis.cli.application.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.exception.error.ErrorLevel; +import org.apache.linkis.cli.application.interactor.job.interactive.InteractiveJobDesc; import org.apache.commons.lang3.StringUtils; import java.util.Map; -public class LinkisSubmitValidator implements Validator { - @Override - public void doValidation(Object input) throws LinkisClientRuntimeException { - if (!(input instanceof LinkisSubmitJob)) { - throw new ValidateException( - "VLD0007", - ErrorLevel.ERROR, - CommonErrMsg.ValidationErr, - "Input of LinkisSubmitValidator is not instance of LinkisSubmitJob. Type: " - + input.getClass().getCanonicalName()); - } +public class InteractiveDescValidator { + + public void doValidation(InteractiveJobDesc desc) throws LinkisClientRuntimeException { boolean ok = true; StringBuilder reasonSb = new StringBuilder(); - LinkisSubmitDesc submitDesc = ((LinkisSubmitJob) input).getJobDesc(); - if (StringUtils.isBlank(submitDesc.getSubmitUser())) { + if (StringUtils.isBlank(desc.getSubmitUser())) { reasonSb.append("Submit User cannot be empty or blank").append(System.lineSeparator()); ok = false; } - if (StringUtils.isBlank(submitDesc.getProxyUser())) { + if (StringUtils.isBlank(desc.getProxyUser())) { reasonSb .append("proxy(execute) User cannot be empty or blank") .append(System.lineSeparator()); ok = false; } - if (submitDesc.getLabelMap() == null) { + if (desc.getLabelMap() == null) { reasonSb.append("labelMap cannot be null").append(System.lineSeparator()); ok = false; } - if (submitDesc.getExecutionMap() == null) { + if (desc.getExecutionMap() == null) { reasonSb.append("ExecutionMap cannot be null").append(System.lineSeparator()); ok = false; } - if (submitDesc.getSourceMap() == null) { + if (desc.getSourceMap() == null) { reasonSb.append("SourceMap cannot be null").append(System.lineSeparator()); ok = false; } - if (submitDesc.getParamConfMap() == null) { + if (desc.getParamConfMap() == null) { reasonSb.append("startupMap cannot be null").append(System.lineSeparator()); ok = false; } - if (submitDesc.getParamVarsMap() == null) { + if (desc.getParamVarsMap() == null) { reasonSb.append("variableMap cannot be null").append(System.lineSeparator()); ok = false; } - if (submitDesc.getParamRunTimeMap() == null) { + if (desc.getParamRunTimeMap() == null) { reasonSb.append("runTimeMap cannot be null").append(System.lineSeparator()); ok = false; } - for (Map.Entry entry : submitDesc.getExecutionMap().entrySet()) { + for (Map.Entry entry : desc.getExecutionMap().entrySet()) { if (StringUtils.contains(entry.getKey(), " ")) { reasonSb .append("ExecutionMap key cannot contains space character. key: ") @@ -87,7 +76,7 @@ public void doValidation(Object input) throws LinkisClientRuntimeException { ok = false; } } - for (Map.Entry entry : submitDesc.getLabelMap().entrySet()) { + for (Map.Entry entry : desc.getLabelMap().entrySet()) { if (StringUtils.contains(entry.getKey(), " ")) { reasonSb .append("LabelMap key cannot contains space character. key: ") @@ -108,7 +97,7 @@ public void doValidation(Object input) throws LinkisClientRuntimeException { } } } - for (Map.Entry entry : submitDesc.getParamConfMap().entrySet()) { + for (Map.Entry entry : desc.getParamConfMap().entrySet()) { if (StringUtils.contains(entry.getKey(), " ")) { reasonSb .append("startUpMap key cannot contains space character. key: ") @@ -145,7 +134,7 @@ public void doValidation(Object input) throws LinkisClientRuntimeException { // } // } // } - for (Map.Entry entry : submitDesc.getParamVarsMap().entrySet()) { + for (Map.Entry entry : desc.getParamVarsMap().entrySet()) { if (StringUtils.contains(entry.getKey(), " ")) { reasonSb .append("variablesMap key cannot contains space character. key: ") @@ -164,7 +153,7 @@ public void doValidation(Object input) throws LinkisClientRuntimeException { // } // } } - for (Map.Entry entry : submitDesc.getSourceMap().entrySet()) { + for (Map.Entry entry : desc.getSourceMap().entrySet()) { if (StringUtils.contains(entry.getKey(), " ")) { reasonSb .append("sourceMap key cannot contains space character. key: ") @@ -185,30 +174,30 @@ public void doValidation(Object input) throws LinkisClientRuntimeException { } } } - if (StringUtils.isBlank((String) submitDesc.getLabelMap().get(LinkisKeys.KEY_ENGINETYPE))) { + if (StringUtils.isBlank((String) desc.getLabelMap().get(LinkisKeys.KEY_ENGINETYPE))) { reasonSb .append(LinkisKeys.KEY_ENGINETYPE) .append(" cannot be empty or blank") .append(System.lineSeparator()); ok = false; } - if (StringUtils.isBlank((String) submitDesc.getLabelMap().get(LinkisKeys.KEY_CODETYPE))) { + if (StringUtils.isBlank((String) desc.getLabelMap().get(LinkisKeys.KEY_CODETYPE))) { reasonSb .append(LinkisKeys.KEY_CODETYPE) .append(" cannot be empty or blank") .append(System.lineSeparator()); ok = false; } - if (StringUtils.isBlank((String) submitDesc.getSourceMap().get(LinkisKeys.KEY_SCRIPT_PATH))) { + if (StringUtils.isBlank((String) desc.getSourceMap().get(LinkisKeys.KEY_SCRIPT_PATH))) { reasonSb .append(LinkisKeys.KEY_SCRIPT_PATH) .append(" cannot be empty or blank") .append(System.lineSeparator()); ok = false; } - if (StringUtils.isBlank((String) submitDesc.getExecutionMap().get(LinkisKeys.KEY_CODE)) + if (StringUtils.isBlank((String) desc.getExecutionMap().get(LinkisKeys.KEY_CODE)) && StringUtils.indexOfIgnoreCase( - (String) submitDesc.getLabelMap().get(LinkisKeys.KEY_ENGINETYPE), "sqoop") + (String) desc.getLabelMap().get(LinkisKeys.KEY_ENGINETYPE), "sqoop") == -1) { reasonSb .append(LinkisKeys.KEY_CODE) @@ -216,15 +205,15 @@ public void doValidation(Object input) throws LinkisClientRuntimeException { .append(System.lineSeparator()); ok = false; } - if (StringUtils.isBlank((String) submitDesc.getLabelMap().get(LinkisKeys.KEY_USER_CREATOR))) { + if (StringUtils.isBlank((String) desc.getLabelMap().get(LinkisKeys.KEY_USER_CREATOR))) { reasonSb .append(LinkisKeys.KEY_USER_CREATOR) .append(" cannot be empty or blank") .append(System.lineSeparator()); ok = false; } else { - String userCreator = (String) submitDesc.getLabelMap().get(LinkisKeys.KEY_USER_CREATOR); - if (StringUtils.indexOf(submitDesc.getProxyUser(), "-") != -1) { + String userCreator = (String) desc.getLabelMap().get(LinkisKeys.KEY_USER_CREATOR); + if (StringUtils.indexOf(desc.getProxyUser(), "-") != -1) { reasonSb .append("\'proxyUser\' should not contain special character \'-\'") .append(System.lineSeparator()); diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/validate/LinkisManageValidator.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/validate/JobCmdDescValidator.java similarity index 60% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/validate/LinkisManageValidator.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/validate/JobCmdDescValidator.java index 519a2ee85c..95b95d5833 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/validate/LinkisManageValidator.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/validate/JobCmdDescValidator.java @@ -17,30 +17,18 @@ package org.apache.linkis.cli.application.interactor.validate; -import org.apache.linkis.cli.application.interactor.job.LinkisManageJob; -import org.apache.linkis.cli.application.interactor.job.desc.LinkisJobManDesc; -import org.apache.linkis.cli.common.entity.validate.Validator; -import org.apache.linkis.cli.common.exception.LinkisClientRuntimeException; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.exception.ValidateException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.exception.LinkisClientRuntimeException; +import org.apache.linkis.cli.application.exception.ValidateException; +import org.apache.linkis.cli.application.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.exception.error.ErrorLevel; +import org.apache.linkis.cli.application.interactor.job.jobcmd.JobCmdDesc; import org.apache.commons.lang3.StringUtils; -public class LinkisManageValidator implements Validator { - @Override - public void doValidation(Object input) throws LinkisClientRuntimeException { - if (!(input instanceof LinkisManageJob)) { - throw new ValidateException( - "VLD0007", - ErrorLevel.ERROR, - CommonErrMsg.ValidationErr, - "Input of LinkisSubmitValidator is not instance of LinkisManageJob. Type: " - + input.getClass().getCanonicalName()); - } +public class JobCmdDescValidator { + public void doValidation(JobCmdDesc desc) throws LinkisClientRuntimeException { boolean ok = true; StringBuilder reasonSb = new StringBuilder(); - LinkisJobManDesc desc = ((LinkisManageJob) input).getJobDesc(); if (StringUtils.isBlank(desc.getJobID())) { reasonSb.append("jobId cannot be empty or blank").append(System.lineSeparator()); ok = false; diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/validate/LinkisOnceSubmitValidator.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/validate/OnceDescValidator.java similarity index 77% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/validate/LinkisOnceSubmitValidator.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/validate/OnceDescValidator.java index fba6644700..f62cafe054 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/validate/LinkisOnceSubmitValidator.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/validate/OnceDescValidator.java @@ -18,71 +18,59 @@ package org.apache.linkis.cli.application.interactor.validate; import org.apache.linkis.cli.application.constants.LinkisKeys; -import org.apache.linkis.cli.application.interactor.job.LinkisOnceJob; -import org.apache.linkis.cli.application.interactor.job.desc.LinkisOnceDesc; -import org.apache.linkis.cli.common.entity.validate.Validator; -import org.apache.linkis.cli.common.exception.LinkisClientRuntimeException; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.exception.ValidateException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.exception.LinkisClientRuntimeException; +import org.apache.linkis.cli.application.exception.ValidateException; +import org.apache.linkis.cli.application.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.exception.error.ErrorLevel; +import org.apache.linkis.cli.application.operator.once.OnceJobDesc; import org.apache.commons.lang3.StringUtils; import java.util.Map; -public class LinkisOnceSubmitValidator implements Validator { - @Override - public void doValidation(Object input) throws LinkisClientRuntimeException { - if (!(input instanceof LinkisOnceJob)) { - throw new ValidateException( - "VLD0007", - ErrorLevel.ERROR, - CommonErrMsg.ValidationErr, - "Input of LinkisSubmitValidator is not instance of LinkisSubmitJob. Type: " - + input.getClass().getCanonicalName()); - } +public class OnceDescValidator { + public void doValidation(OnceJobDesc desc) throws LinkisClientRuntimeException { boolean ok = true; StringBuilder reasonSb = new StringBuilder(); - LinkisOnceDesc submitDesc = (LinkisOnceDesc) ((LinkisOnceJob) input).getJobDesc(); - if (StringUtils.isBlank(submitDesc.getSubmitUser())) { + if (StringUtils.isBlank(desc.getSubmitUser())) { reasonSb.append("Submit User cannot be empty or blank").append(System.lineSeparator()); ok = false; } - if (StringUtils.isBlank(submitDesc.getProxyUser())) { + if (StringUtils.isBlank(desc.getProxyUser())) { reasonSb .append("proxy(execute) User cannot be empty or blank") .append(System.lineSeparator()); ok = false; } - if (submitDesc.getLabelMap() == null) { + if (desc.getLabelMap() == null) { reasonSb.append("labelMap cannot be null").append(System.lineSeparator()); ok = false; } - if (submitDesc.getExecutionMap() == null) { + if (desc.getExecutionMap() == null) { reasonSb.append("ExecutionMap cannot be null").append(System.lineSeparator()); ok = false; } - if (submitDesc.getSourceMap() == null) { + if (desc.getSourceMap() == null) { reasonSb.append("SourceMap cannot be null").append(System.lineSeparator()); ok = false; } - if (submitDesc.getParamConfMap() == null) { + if (desc.getParamConfMap() == null) { reasonSb.append("startupMap cannot be null").append(System.lineSeparator()); ok = false; } - if (submitDesc.getParamVarsMap() == null) { + if (desc.getParamVarsMap() == null) { reasonSb.append("variableMap cannot be null").append(System.lineSeparator()); ok = false; } - if (submitDesc.getParamRunTimeMap() == null) { + if (desc.getParamRunTimeMap() == null) { reasonSb.append("runTimeMap cannot be null").append(System.lineSeparator()); ok = false; } - if (submitDesc.getJobContentMap() == null) { + if (desc.getJobContentMap() == null) { reasonSb.append("jobContentMap cannot be null").append(System.lineSeparator()); ok = false; } - for (Map.Entry entry : submitDesc.getExecutionMap().entrySet()) { + for (Map.Entry entry : desc.getExecutionMap().entrySet()) { if (StringUtils.contains(entry.getKey(), " ")) { reasonSb .append("ExecutionMap key cannot contains space character. key: ") @@ -91,7 +79,7 @@ public void doValidation(Object input) throws LinkisClientRuntimeException { ok = false; } } - for (Map.Entry entry : submitDesc.getLabelMap().entrySet()) { + for (Map.Entry entry : desc.getLabelMap().entrySet()) { if (StringUtils.contains(entry.getKey(), " ")) { reasonSb .append("LabelMap key cannot contains space character. key: ") @@ -112,7 +100,7 @@ public void doValidation(Object input) throws LinkisClientRuntimeException { } } } - for (Map.Entry entry : submitDesc.getParamConfMap().entrySet()) { + for (Map.Entry entry : desc.getParamConfMap().entrySet()) { if (StringUtils.contains(entry.getKey(), " ")) { reasonSb .append("startUpMap key cannot contains space character. key: ") @@ -149,7 +137,7 @@ public void doValidation(Object input) throws LinkisClientRuntimeException { // } // } // } - for (Map.Entry entry : submitDesc.getParamVarsMap().entrySet()) { + for (Map.Entry entry : desc.getParamVarsMap().entrySet()) { if (StringUtils.contains(entry.getKey(), " ")) { reasonSb .append("variablesMap key cannot contains space character. key: ") @@ -168,7 +156,7 @@ public void doValidation(Object input) throws LinkisClientRuntimeException { // } // } } - for (Map.Entry entry : submitDesc.getSourceMap().entrySet()) { + for (Map.Entry entry : desc.getSourceMap().entrySet()) { if (StringUtils.contains(entry.getKey(), " ")) { reasonSb .append("sourceMap key cannot contains space character. key: ") @@ -189,7 +177,7 @@ public void doValidation(Object input) throws LinkisClientRuntimeException { } } } - for (Map.Entry entry : submitDesc.getJobContentMap().entrySet()) { + for (Map.Entry entry : desc.getJobContentMap().entrySet()) { if (StringUtils.contains(entry.getKey(), " ")) { reasonSb .append("jobContentMap key cannot contains space character. key: ") @@ -208,30 +196,30 @@ public void doValidation(Object input) throws LinkisClientRuntimeException { // } // } } - if (StringUtils.isBlank((String) submitDesc.getLabelMap().get(LinkisKeys.KEY_ENGINETYPE))) { + if (StringUtils.isBlank((String) desc.getLabelMap().get(LinkisKeys.KEY_ENGINETYPE))) { reasonSb .append(LinkisKeys.KEY_ENGINETYPE) .append(" cannot be empty or blank") .append(System.lineSeparator()); ok = false; } - if (StringUtils.isBlank((String) submitDesc.getLabelMap().get(LinkisKeys.KEY_CODETYPE))) { + if (StringUtils.isBlank((String) desc.getLabelMap().get(LinkisKeys.KEY_CODETYPE))) { reasonSb .append(LinkisKeys.KEY_CODETYPE) .append(" cannot be empty or blank") .append(System.lineSeparator()); ok = false; } - if (StringUtils.isBlank((String) submitDesc.getSourceMap().get(LinkisKeys.KEY_SCRIPT_PATH))) { + if (StringUtils.isBlank((String) desc.getSourceMap().get(LinkisKeys.KEY_SCRIPT_PATH))) { reasonSb .append(LinkisKeys.KEY_SCRIPT_PATH) .append(" cannot be empty or blank") .append(System.lineSeparator()); ok = false; } - if (StringUtils.isBlank((String) submitDesc.getExecutionMap().get(LinkisKeys.KEY_CODE)) + if (StringUtils.isBlank((String) desc.getExecutionMap().get(LinkisKeys.KEY_CODE)) && StringUtils.indexOfIgnoreCase( - (String) submitDesc.getLabelMap().get(LinkisKeys.KEY_ENGINETYPE), "sqoop") + (String) desc.getLabelMap().get(LinkisKeys.KEY_ENGINETYPE), "sqoop") == -1) { reasonSb .append(LinkisKeys.KEY_CODE) @@ -239,15 +227,15 @@ public void doValidation(Object input) throws LinkisClientRuntimeException { .append(System.lineSeparator()); ok = false; } - if (StringUtils.isBlank((String) submitDesc.getLabelMap().get(LinkisKeys.KEY_USER_CREATOR))) { + if (StringUtils.isBlank((String) desc.getLabelMap().get(LinkisKeys.KEY_USER_CREATOR))) { reasonSb .append(LinkisKeys.KEY_USER_CREATOR) .append(" cannot be empty or blank") .append(System.lineSeparator()); ok = false; } else { - String userCreator = (String) submitDesc.getLabelMap().get(LinkisKeys.KEY_USER_CREATOR); - if (StringUtils.indexOf(submitDesc.getProxyUser(), "-") != -1) { + String userCreator = (String) desc.getLabelMap().get(LinkisKeys.KEY_USER_CREATOR); + if (StringUtils.indexOf(desc.getProxyUser(), "-") != -1) { reasonSb .append("\'proxyUser\' should not contain special character \'-\'") .append(System.lineSeparator()); diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/validate/ParamValidator.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/validate/ParamValidator.java similarity index 75% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/validate/ParamValidator.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/validate/ParamValidator.java index abbc695976..b54dd6b723 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/validate/ParamValidator.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/validate/ParamValidator.java @@ -15,14 +15,12 @@ * limitations under the License. */ -package org.apache.linkis.cli.core.interactor.validate; +package org.apache.linkis.cli.application.interactor.validate; -import org.apache.linkis.cli.common.entity.command.Params; -import org.apache.linkis.cli.common.entity.validate.Validator; -import org.apache.linkis.cli.common.exception.LinkisClientRuntimeException; +import org.apache.linkis.cli.application.entity.command.Params; +import org.apache.linkis.cli.application.exception.LinkisClientRuntimeException; -public class ParamValidator implements Validator { - @Override +public class ParamValidator { public void doValidation(Object input) throws LinkisClientRuntimeException { if (!(input instanceof Params)) { // TODO:throw diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/validate/ParsedTplValidator.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/validate/ParsedTplValidator.java similarity index 69% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/validate/ParsedTplValidator.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/validate/ParsedTplValidator.java index e7f14e2df1..a7ea05c72b 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/validate/ParsedTplValidator.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/validate/ParsedTplValidator.java @@ -15,15 +15,14 @@ * limitations under the License. */ -package org.apache.linkis.cli.core.interactor.validate; +package org.apache.linkis.cli.application.interactor.validate; -import org.apache.linkis.cli.common.entity.command.CmdOption; -import org.apache.linkis.cli.common.entity.command.CmdTemplate; -import org.apache.linkis.cli.common.entity.validate.Validator; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.exception.CommandException; -import org.apache.linkis.cli.core.exception.ValidateException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.entity.command.CmdOption; +import org.apache.linkis.cli.application.entity.command.CmdTemplate; +import org.apache.linkis.cli.application.exception.CommandException; +import org.apache.linkis.cli.application.exception.ValidateException; +import org.apache.linkis.cli.application.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.exception.error.ErrorLevel; import java.text.MessageFormat; import java.util.List; @@ -35,20 +34,10 @@ * 1. Check if there is missing or unknown option. 2. Call checkParam method for command-specific * validation. */ -public class ParsedTplValidator implements Validator { +public class ParsedTplValidator { private static final Logger logger = LoggerFactory.getLogger(ParsedTplValidator.class); - @Override - public void doValidation(Object input) throws CommandException { - if (!(input instanceof CmdTemplate)) { - throw new ValidateException( - "VLD0006", - ErrorLevel.ERROR, - CommonErrMsg.ValidationErr, - "Input of ParsedTplValidator is not instance of CmdTemplate"); - } - - CmdTemplate parsedTemplateCopy = (CmdTemplate) input; + public void doValidation(CmdTemplate parsedTemplateCopy) throws CommandException { String msg = "start validating command \"{0}\", template \"{1}\""; logger.info( diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/validate/UJESContextValidator.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/validate/UJESContextValidator.java similarity index 75% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/validate/UJESContextValidator.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/validate/UJESContextValidator.java index 8c0653cb4f..9e5a9a6457 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/validate/UJESContextValidator.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/validate/UJESContextValidator.java @@ -18,28 +18,18 @@ package org.apache.linkis.cli.application.interactor.validate; import org.apache.linkis.cli.application.constants.LinkisConstants; +import org.apache.linkis.cli.application.exception.LinkisClientRuntimeException; +import org.apache.linkis.cli.application.exception.ValidateException; +import org.apache.linkis.cli.application.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.exception.error.ErrorLevel; import org.apache.linkis.cli.application.operator.ujes.UJESClientContext; -import org.apache.linkis.cli.common.entity.validate.Validator; -import org.apache.linkis.cli.common.exception.LinkisClientRuntimeException; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.exception.ValidateException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; import org.apache.commons.lang3.StringUtils; -public class UJESContextValidator implements Validator { - @Override - public void doValidation(Object input) throws LinkisClientRuntimeException { - if (!(input instanceof UJESClientContext)) { - throw new ValidateException( - "VLD0009", - ErrorLevel.ERROR, - CommonErrMsg.ValidationErr, - "Input of UJESContextValidator is not instance of UjesClientDriverContext"); - } +public class UJESContextValidator { + public void doValidation(UJESClientContext context) throws LinkisClientRuntimeException { boolean ok = true; StringBuilder reasonSb = new StringBuilder(); - UJESClientContext context = (UJESClientContext) input; if (StringUtils.isBlank(context.getGatewayUrl())) { reasonSb.append("gatewayUrl cannot be empty or blank").append(System.lineSeparator()); ok = false; diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/var/StdVarAccess.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/var/VarAccessImpl.java similarity index 88% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/var/StdVarAccess.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/var/VarAccessImpl.java index 267dae2325..9a871828a3 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/var/StdVarAccess.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/var/VarAccessImpl.java @@ -15,18 +15,18 @@ * limitations under the License. */ -package org.apache.linkis.cli.core.interactor.var; - -import org.apache.linkis.cli.common.entity.command.ParamItem; -import org.apache.linkis.cli.common.entity.command.Params; -import org.apache.linkis.cli.common.entity.properties.ClientProperties; -import org.apache.linkis.cli.common.entity.var.VarAccess; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.exception.VarAccessException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; -import org.apache.linkis.cli.core.interactor.command.SpecialMap; -import org.apache.linkis.cli.core.utils.converter.AbstractStringConverter; -import org.apache.linkis.cli.core.utils.converter.PredefinedStringConverters; +package org.apache.linkis.cli.application.interactor.var; + +import org.apache.linkis.cli.application.entity.command.ParamItem; +import org.apache.linkis.cli.application.entity.command.Params; +import org.apache.linkis.cli.application.entity.var.VarAccess; +import org.apache.linkis.cli.application.exception.VarAccessException; +import org.apache.linkis.cli.application.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.exception.error.ErrorLevel; +import org.apache.linkis.cli.application.interactor.command.SpecialMap; +import org.apache.linkis.cli.application.interactor.command.template.converter.AbstractStringConverter; +import org.apache.linkis.cli.application.interactor.command.template.converter.PredefinedStringConverters; +import org.apache.linkis.cli.application.interactor.properties.ClientProperties; import org.apache.commons.lang3.StringUtils; @@ -35,14 +35,14 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class StdVarAccess implements VarAccess { - private static Logger logger = LoggerFactory.getLogger(StdVarAccess.class); +public class VarAccessImpl implements VarAccess { + private static Logger logger = LoggerFactory.getLogger(VarAccessImpl.class); private Params cmdParams; private ClientProperties userConf; private ClientProperties defaultConf; private Map subMapCache; - public StdVarAccess setCmdParams(Params cmdParams) { + public VarAccessImpl setCmdParams(Params cmdParams) { this.cmdParams = cmdParams; return this; } @@ -51,7 +51,7 @@ public Params getSubParam(String identifier) { return this.cmdParams; } - public StdVarAccess setUserConf(ClientProperties userConf) { + public VarAccessImpl setUserConf(ClientProperties userConf) { this.userConf = userConf; return this; } @@ -60,7 +60,7 @@ public ClientProperties getUserConf(String identifier) { return this.userConf; } - public StdVarAccess setDefaultConf(ClientProperties defaultConf) { + public VarAccessImpl setDefaultConf(ClientProperties defaultConf) { this.defaultConf = defaultConf; return this; } @@ -69,7 +69,7 @@ public ClientProperties getDefaultConf(String identifier) { return this.defaultConf; } - public StdVarAccess init() { + public VarAccessImpl init() { this.subMapCache = new HashMap<>(); putSubMapCache(subMapCache, cmdParams); return this; @@ -104,7 +104,6 @@ private void putSubMapCache(Map subMapCache, Params param) { } } - @Override public void checkInit() { if (this.cmdParams == null || this.defaultConf == null || this.subMapCache == null) { throw new VarAccessException( diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/observer/event/TriggerEvent.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/observer/event/FetchResultEvent.java similarity index 93% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/observer/event/TriggerEvent.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/observer/event/FetchResultEvent.java index dada521b59..d538a40dff 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/observer/event/TriggerEvent.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/observer/event/FetchResultEvent.java @@ -17,4 +17,4 @@ package org.apache.linkis.cli.application.observer.event; -public class TriggerEvent extends SingleObserverEvent {} +public class FetchResultEvent extends SingleObserverEvent {} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/observer/event/LinkisClientEvent.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/observer/event/LinkisClientEvent.java similarity index 100% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/observer/event/LinkisClientEvent.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/observer/event/LinkisClientEvent.java diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/observer/event/LogStartEvent.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/observer/event/LogStartEvent.java similarity index 100% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/observer/event/LogStartEvent.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/observer/event/LogStartEvent.java diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/observer/event/SingleObserverEvent.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/observer/event/SingleObserverEvent.java similarity index 100% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/observer/event/SingleObserverEvent.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/observer/event/SingleObserverEvent.java diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/observer/listener/LinkisClientListener.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/observer/listener/LinkisClientListener.java similarity index 100% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/observer/listener/LinkisClientListener.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/observer/listener/LinkisClientListener.java diff --git a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/JobOperBuilder.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/JobOperBuilder.java new file mode 100644 index 0000000000..0e1604b204 --- /dev/null +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/JobOperBuilder.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.cli.application.operator; + +import org.apache.linkis.cli.application.entity.context.CliCtx; +import org.apache.linkis.cli.application.entity.operator.JobOper; + +public interface JobOperBuilder { + JobOper build(CliCtx ctx); +} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/OperManager.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/OperManager.java new file mode 100644 index 0000000000..a41bfd5f9f --- /dev/null +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/OperManager.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.cli.application.operator; + +import org.apache.linkis.cli.application.entity.context.CliCtx; +import org.apache.linkis.cli.application.entity.operator.JobOper; + +import java.util.HashMap; +import java.util.Map; + +public class OperManager { + + private static Map builderMap = new HashMap<>(); + + public static void register(String name, JobOperBuilder builder) { + builderMap.put(name, builder); + } + + public static void remove(String name) { + builderMap.remove(name); + } + + public static JobOper getNew(String name, CliCtx ctx) { + JobOperBuilder builder = builderMap.get(name); + if (builder == null) { + return null; + } + return builder.build(ctx); + } +} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/operator/once/LinkisNodeStatus.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/once/LinkisNodeStatus.java similarity index 100% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/operator/once/LinkisNodeStatus.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/once/LinkisNodeStatus.java diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/operator/once/OnceJobConstants.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/once/OnceJobConstants.java similarity index 100% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/operator/once/OnceJobConstants.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/once/OnceJobConstants.java diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/desc/LinkisOnceDesc.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/once/OnceJobDesc.java similarity index 82% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/desc/LinkisOnceDesc.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/once/OnceJobDesc.java index a98d99383e..2bb63c0e0e 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/desc/LinkisOnceDesc.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/once/OnceJobDesc.java @@ -15,18 +15,14 @@ * limitations under the License. */ -package org.apache.linkis.cli.application.interactor.job.desc; +package org.apache.linkis.cli.application.operator.once; -import org.apache.linkis.cli.common.entity.job.JobDescription; -import org.apache.linkis.cli.common.entity.var.VarAccess; +import org.apache.linkis.cli.application.entity.var.VarAccess; import java.util.Map; -public class LinkisOnceDesc implements JobDescription { - - private VarAccess stdVarAccess; - private VarAccess sysVarAccess; - +public class OnceJobDesc { + private VarAccess varAccess; private String submitUser; private String proxyUser; private String creator; @@ -38,20 +34,12 @@ public class LinkisOnceDesc implements JobDescription { private Map sourceMap; private Map jobContentMap; - public VarAccess getStdVarAccess() { - return stdVarAccess; - } - - public void setStdVarAccess(VarAccess stdVarAccess) { - this.stdVarAccess = stdVarAccess; - } - - public VarAccess getSysVarAccess() { - return sysVarAccess; + public VarAccess getVarAccess() { + return varAccess; } - public void setSysVarAccess(VarAccess sysVarAccess) { - this.sysVarAccess = sysVarAccess; + public void setVarAccess(VarAccess varAccess) { + this.varAccess = varAccess; } public String getSubmitUser() { diff --git a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/once/OnceJobOper.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/once/OnceJobOper.java new file mode 100644 index 0000000000..afe2f66996 --- /dev/null +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/once/OnceJobOper.java @@ -0,0 +1,170 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.cli.application.operator.once; + +import org.apache.linkis.cli.application.constants.CliConstants; +import org.apache.linkis.cli.application.constants.CliKeys; +import org.apache.linkis.cli.application.constants.LinkisKeys; +import org.apache.linkis.cli.application.entity.job.JobStatus; +import org.apache.linkis.cli.application.entity.operator.JobOper; +import org.apache.linkis.cli.application.entity.var.VarAccess; +import org.apache.linkis.cli.application.exception.LinkisClientExecutionException; +import org.apache.linkis.cli.application.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.exception.error.ErrorLevel; +import org.apache.linkis.cli.application.interactor.job.common.LinkisJobStatus; +import org.apache.linkis.cli.application.operator.ujes.UJESClientFactory; +import org.apache.linkis.computation.client.LinkisJobBuilder$; +import org.apache.linkis.computation.client.once.simple.SimpleOnceJob; +import org.apache.linkis.computation.client.once.simple.SimpleOnceJobBuilder; +import org.apache.linkis.computation.client.once.simple.SubmittableSimpleOnceJob; +import org.apache.linkis.computation.client.operator.impl.EngineConnLogOperator; +import org.apache.linkis.computation.client.operator.impl.EngineConnLogs; + +import org.apache.commons.lang3.StringUtils; + +public class OnceJobOper implements JobOper { + + EngineConnLogOperator logOperator = null; + private SimpleOnceJob onceJob; + private String serverUrl; + private String engineTypeForECM; + private Boolean isLogFin = false; + + public void init(OnceJobDesc desc) { + + VarAccess varAccess = desc.getVarAccess(); + + serverUrl = varAccess.getVar(String.class, CliKeys.LINKIS_COMMON_GATEWAY_URL); + + LinkisJobBuilder$.MODULE$.setDefaultClientConfig( + UJESClientFactory.generateDWSClientConfig(varAccess)); + LinkisJobBuilder$.MODULE$.setDefaultUJESClient(UJESClientFactory.getReusable(varAccess)); + + String engineTypeRaw = (String) desc.getLabelMap().get(LinkisKeys.KEY_ENGINETYPE); + engineTypeForECM = engineTypeRaw; + + if (StringUtils.isNotBlank(engineTypeRaw)) { + engineTypeForECM = StringUtils.split(engineTypeRaw, "-")[0]; + } else { + engineTypeForECM = ""; + } // TODO: remove parsing and let server side parse engineType + + onceJob = + new SimpleOnceJobBuilder() + .setCreateService(CliConstants.LINKIS_CLI) + .addExecuteUser(desc.getProxyUser()) + .setStartupParams(desc.getParamConfMap()) + .setLabels(desc.getLabelMap()) + .setRuntimeParams(desc.getParamRunTimeMap()) + .setSource(desc.getSourceMap()) + .setVariableMap(desc.getParamVarsMap()) + .setJobContent(desc.getJobContentMap()) + .build(); + } + + public String getServerUrl() { + return serverUrl; + } + + public SimpleOnceJob getOnceJob() { + return onceJob; + } + + public void setOnceJob(SimpleOnceJob onceJob) { + this.onceJob = onceJob; + } + + private void panicIfNull(Object obj) { + if (obj == null) { + throw new LinkisClientExecutionException( + "EXE0040", + ErrorLevel.ERROR, + CommonErrMsg.ExecutionErr, + "Instance of " + obj.getClass().getCanonicalName() + " is null"); + } + } + + public void submit() { + panicIfNull(onceJob); + if (!(onceJob instanceof SubmittableSimpleOnceJob)) { + throw new LinkisClientExecutionException( + "EXE0041", + ErrorLevel.ERROR, + CommonErrMsg.ExecutionErr, + "onceJob is not properly initiated"); + } + ((SubmittableSimpleOnceJob) onceJob).submit(); + } + + public void kill() { + panicIfNull(onceJob); + if (!getStatus().isJobFinishedState()) { + onceJob.kill(); + } + } + + public String getJobID() { + return onceJob.getId(); + } + + public String getUser() { + return "TODO"; + } + + public JobStatus getStatus() { + panicIfNull(onceJob); + String status = onceJob.getStatus(); + return LinkisJobStatus.convertFromNodeStatusString(status); + } + + public void waitForComplete() { + panicIfNull(onceJob); + onceJob.waitForCompleted(); + } + + public String getCurrentLog() { + panicIfNull(onceJob); + if (logOperator == null) { + logOperator = + (EngineConnLogOperator) onceJob.getOperator(EngineConnLogOperator.OPERATOR_NAME()); + logOperator.setECMServiceInstance( + ((SubmittableSimpleOnceJob) onceJob).getECMServiceInstance()); + logOperator.setEngineConnType(engineTypeForECM); + // logOperator.setPageSize(OnceJobConstants.MAX_LOG_SIZE_ONCE); + logOperator.setIgnoreKeywords(OnceJobConstants.LOG_IGNORE_KEYWORDS); + } + EngineConnLogs logs = + (EngineConnLogs) logOperator.apply(); // for some reason we have to add type conversion, + // otherwise mvn testCompile fails + StringBuilder logBuilder = new StringBuilder(); + for (String log : logs.logs()) { + logBuilder.append(log).append(System.lineSeparator()); + } + String status = onceJob.getStatus(); + LinkisJobStatus jobStatus = LinkisJobStatus.convertFromNodeStatusString(status); + if ((logs.logs() == null || logs.logs().size() <= 0) && jobStatus.isJobFinishedState()) { + isLogFin = true; + } + return logBuilder.toString(); + // System.out.println(logs.logs().size()); + } + + public Boolean isLogFin() { + return isLogFin; + } +} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/once/OnceOperBuilder.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/once/OnceOperBuilder.java new file mode 100644 index 0000000000..906fc03406 --- /dev/null +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/once/OnceOperBuilder.java @@ -0,0 +1,170 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.cli.application.operator.once; + +import org.apache.linkis.cli.application.constants.CliConstants; +import org.apache.linkis.cli.application.constants.CliKeys; +import org.apache.linkis.cli.application.constants.LinkisKeys; +import org.apache.linkis.cli.application.entity.context.CliCtx; +import org.apache.linkis.cli.application.entity.operator.JobOper; +import org.apache.linkis.cli.application.interactor.job.common.KeyParser; +import org.apache.linkis.cli.application.operator.JobOperBuilder; +import org.apache.linkis.cli.application.utils.CliUtils; + +import org.apache.commons.lang3.StringUtils; + +import java.util.*; + +public class OnceOperBuilder implements JobOperBuilder { + @Override + public JobOper build(CliCtx ctx) { + + OnceJobDesc desc = new OnceJobDesc(); + + Map confMap = ctx.getVarAccess().getVar(Map.class, CliKeys.JOB_PARAM_CONF); + Map runtimeMap = + ctx.getVarAccess().getVar(Map.class, CliKeys.JOB_PARAM_RUNTIME); + Map varMap = ctx.getVarAccess().getVar(Map.class, CliKeys.JOB_PARAM_VAR); + Map labelMap = ctx.getVarAccess().getVar(Map.class, CliKeys.JOB_LABEL); + Map sourceMap = ctx.getVarAccess().getVar(Map.class, CliKeys.JOB_SOURCE); + Map executionMap = ctx.getVarAccess().getVar(Map.class, CliKeys.JOB_EXEC); + Map jobContentMap = ctx.getVarAccess().getVar(Map.class, CliKeys.JOB_CONTENT); + + confMap = confMap == null ? new HashMap<>() : confMap; + runtimeMap = runtimeMap == null ? new HashMap<>() : runtimeMap; + varMap = varMap == null ? new HashMap<>() : varMap; + labelMap = labelMap == null ? new HashMap<>() : labelMap; + sourceMap = sourceMap == null ? new HashMap<>() : sourceMap; + executionMap = executionMap == null ? new HashMap<>() : executionMap; + jobContentMap = jobContentMap == null ? new HashMap<>() : jobContentMap; + + confMap = KeyParser.removePrefixForKeysInMap(confMap); + runtimeMap = KeyParser.removePrefixForKeysInMap(runtimeMap); + labelMap = KeyParser.removePrefixForKeysInMap(labelMap); + sourceMap = KeyParser.removePrefixForKeysInMap(sourceMap); + executionMap = KeyParser.removePrefixForKeysInMap(executionMap); + jobContentMap = KeyParser.removePrefixForKeysInMap(jobContentMap); + + for (String key : ctx.getVarAccess().getAllVarKeys()) { + Object val = ctx.getVarAccess().getVar(Object.class, key); + if (!(val instanceof Map) && val != null) { + // note that we allow it to overwrite existing values in map + if (StringUtils.startsWithIgnoreCase(key, CliKeys.JOB_PARAM_CONF)) { + KeyParser.removePrefixAndPutValToMap(confMap, key, val, CliKeys.JOB_PARAM_CONF); + } else if (StringUtils.startsWithIgnoreCase(key, CliKeys.JOB_PARAM_VAR)) { + KeyParser.removePrefixAndPutValToMap(varMap, key, val, CliKeys.JOB_PARAM_VAR); + } else if (StringUtils.startsWithIgnoreCase(key, CliKeys.JOB_PARAM_RUNTIME)) { + KeyParser.removePrefixAndPutValToMap(runtimeMap, key, val, CliKeys.JOB_PARAM_RUNTIME); + } else if (StringUtils.startsWithIgnoreCase(key, CliKeys.JOB_EXEC)) { + KeyParser.removePrefixAndPutValToMap(executionMap, key, val, CliKeys.JOB_EXEC); + } else if (StringUtils.startsWithIgnoreCase(key, CliKeys.JOB_LABEL)) { + KeyParser.removePrefixAndPutValToMap(labelMap, key, val, CliKeys.JOB_LABEL); + } else if (StringUtils.startsWithIgnoreCase(key, CliKeys.JOB_SOURCE)) { + KeyParser.removePrefixAndPutValToMap(sourceMap, key, val, CliKeys.JOB_SOURCE); + } else if (StringUtils.startsWithIgnoreCase(key, CliKeys.JOB_CONTENT)) { + KeyParser.removePrefixAndPutValToMap(jobContentMap, key, val, CliKeys.JOB_CONTENT); + } else if (StringUtils.startsWithIgnoreCase(key, CliKeys.LINKIS_CLIENT_COMMON)) { + // do nothing + } else { + // confMap.put(key, stdVarAccess.getVar(Object.class, key)); + } + } + } + + Boolean isAsync = + ctx.getVarAccess().getVarOrDefault(Boolean.class, CliKeys.LINKIS_CLIENT_ASYNC_OPT, false); + + String creator; + if (!isAsync) { + creator = + ctx.getVarAccess() + .getVarOrDefault( + String.class, CliKeys.JOB_COMMON_CREATOR, CliConstants.JOB_CREATOR_DEFAULT); + } else { + creator = + ctx.getVarAccess() + .getVarOrDefault( + String.class, CliKeys.JOB_COMMON_CREATOR, CliConstants.JOB_CREATOR_ASYNC_DEFAULT); + } + String code = ctx.getVarAccess().getVar(String.class, CliKeys.JOB_EXEC_CODE); + String engineType = ctx.getVarAccess().getVar(String.class, CliKeys.JOB_LABEL_ENGINE_TYPE); + String runType = ctx.getVarAccess().getVar(String.class, CliKeys.JOB_LABEL_CODE_TYPE); + String scriptPath = + ctx.getVarAccess() + .getVarOrDefault(String.class, CliKeys.JOB_SOURCE_SCRIPT_PATH, "LinkisCli"); + + String osUser = System.getProperty(CliKeys.LINUX_USER_KEY); + String[] adminUsers = StringUtils.split(CliKeys.ADMIN_USERS, ','); + Set adminSet = new HashSet<>(); + for (String admin : adminUsers) { + adminSet.add(admin); + } + String submitUsr = CliUtils.getSubmitUser(ctx.getVarAccess(), osUser, adminSet); + String proxyUsr = CliUtils.getProxyUser(ctx.getVarAccess(), submitUsr, adminSet); + + String enableExecuteOnce = + ctx.getVarAccess().getVarOrDefault(String.class, CliKeys.JOB_LABEL_EXECUTEONCE, "true"); + // default executeOnce-mode + if (Boolean.parseBoolean(enableExecuteOnce)) { + labelMap.put(LinkisKeys.KEY_EXECUTEONCE, ""); + } else { + labelMap.remove(LinkisKeys.KEY_EXECUTEONCE); + } + String codePath = ctx.getVarAccess().getVar(String.class, CliKeys.JOB_COMMON_CODE_PATH); + Object extraArgsObj = ctx.getVarAccess().getVar(Object.class, CliKeys.JOB_EXTRA_ARGUMENTS); + if (extraArgsObj != null + && extraArgsObj instanceof String[] + && StringUtils.isBlank(code) + && StringUtils.isBlank(codePath)) { + String[] extraArgs = (String[]) extraArgsObj; + codePath = extraArgs[0]; + if (extraArgs.length > 1) { + runtimeMap.put( + LinkisKeys.EXTRA_ARGUMENTS, Arrays.copyOfRange(extraArgs, 1, extraArgs.length)); + } + } + + if (StringUtils.isBlank(code) && StringUtils.isNotBlank(codePath)) { + code = CliUtils.readFile(codePath); + } + + executionMap.put(LinkisKeys.KEY_CODE, code); + labelMap.put(LinkisKeys.KEY_ENGINETYPE, engineType); + labelMap.put(LinkisKeys.KEY_CODETYPE, runType); + labelMap.put(LinkisKeys.KEY_USER_CREATOR, proxyUsr + "-" + creator); + sourceMap.put(LinkisKeys.KEY_SCRIPT_PATH, scriptPath); + runtimeMap.put(LinkisKeys.KEY_HIVE_RESULT_DISPLAY_TBALE, true); + + desc.setVarAccess(ctx.getVarAccess()); + desc.setCreator(creator); + desc.setParamConfMap(confMap); + desc.setParamRunTimeMap(runtimeMap); + desc.setParamVarsMap(varMap); + desc.setLabelMap(labelMap); + desc.setSourceMap(sourceMap); + desc.setExecutionMap(executionMap); + desc.setSubmitUser(submitUsr); + desc.setProxyUser(proxyUsr); + desc.setJobContentMap(jobContentMap); + + OnceJobOper onceJobOper = new OnceJobOper(); + onceJobOper.init(desc); + + return onceJobOper; + } +} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/operator/ujes/LinkisJobOperator.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/ujes/LinkisJobOper.java similarity index 88% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/operator/ujes/LinkisJobOperator.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/ujes/LinkisJobOper.java index 87c49cd141..191061332b 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/operator/ujes/LinkisJobOperator.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/ujes/LinkisJobOper.java @@ -18,15 +18,15 @@ package org.apache.linkis.cli.application.operator.ujes; import org.apache.linkis.cli.application.constants.LinkisKeys; -import org.apache.linkis.cli.application.interactor.job.desc.LinkisSubmitDesc; +import org.apache.linkis.cli.application.entity.operator.JobOper; +import org.apache.linkis.cli.application.exception.LinkisClientExecutionException; +import org.apache.linkis.cli.application.exception.LinkisClientRuntimeException; +import org.apache.linkis.cli.application.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.exception.error.ErrorLevel; +import org.apache.linkis.cli.application.interactor.job.interactive.InteractiveJobDesc; import org.apache.linkis.cli.application.operator.ujes.result.OpenLogResult2; import org.apache.linkis.cli.application.operator.ujes.result.ResultSetResult2; -import org.apache.linkis.cli.application.utils.Utils; -import org.apache.linkis.cli.common.entity.operator.JobOperator; -import org.apache.linkis.cli.common.exception.LinkisClientRuntimeException; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.exception.LinkisClientExecutionException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.utils.CliUtils; import org.apache.linkis.common.exception.LinkisException; import org.apache.linkis.ujes.client.UJESClient; import org.apache.linkis.ujes.client.request.JobSubmitAction; @@ -43,9 +43,9 @@ import org.slf4j.LoggerFactory; /** Based on UjesClient */ -public class LinkisJobOperator implements JobOperator { +public class LinkisJobOper implements JobOper { protected UJESClient client; - private Logger logger = LoggerFactory.getLogger(LinkisJobOperator.class); + private Logger logger = LoggerFactory.getLogger(LinkisJobOper.class); private String serverUrl; public UJESClient getUJESClient() { @@ -87,7 +87,7 @@ public void checkInit() throws LinkisClientRuntimeException { * @param * @return */ - public LinkisOperResultAdapter submit(LinkisSubmitDesc jobDesc) + public LinkisOperResultAdapter submit(InteractiveJobDesc jobDesc) throws LinkisClientRuntimeException { checkInit(); JobSubmitResult jobSubmitResult; @@ -104,7 +104,7 @@ public LinkisOperResultAdapter submit(LinkisSubmitDesc jobDesc) .setLabels(jobDesc.getLabelMap()) .setSource(jobDesc.getSourceMap()) .build(); - logger.info("Request info to Linkis: \n{}", Utils.GSON.toJson(jobSubmitAction)); + logger.info("Request info to Linkis: \n{}", CliUtils.GSON.toJson(jobSubmitAction)); /* Old API */ // JobExecuteAction jobExecuteAction = JobExecuteAction.builder() @@ -127,7 +127,7 @@ public LinkisOperResultAdapter submit(LinkisSubmitDesc jobDesc) // jobExecuteResult = client.execute(jobExecuteAction); jobSubmitResult = client.submit(jobSubmitAction); - logger.info("Response info from Linkis: \n{}", Utils.GSON.toJson(jobSubmitAction)); + logger.info("Response info from Linkis: \n{}", CliUtils.GSON.toJson(jobSubmitAction)); } catch (Exception e) { // must throw if exception @@ -142,7 +142,8 @@ public LinkisOperResultAdapter submit(LinkisSubmitDesc jobDesc) if (jobSubmitResult == null) { reason = "JobSubmitResult is null"; } else if (0 != jobSubmitResult.getStatus()) { - reason = "server returns non-zero status-code"; + reason = "server returns non-zero status-code. "; + reason += jobSubmitResult.getMessage(); } else { reason = "server returns blank TaskId"; } @@ -191,13 +192,14 @@ public LinkisOperResultAdapter queryJobStatus(String user, String taskID, String while (retryTime++ < MAX_RETRY_TIME) { try { jobStatusResult = client.status(executeResult); - logger.debug("job-status: " + Utils.GSON.toJson(jobStatusResult)); + logger.debug("job-status: " + CliUtils.GSON.toJson(jobStatusResult)); if (jobStatusResult == null || 0 != jobStatusResult.getStatus()) { String reason; if (jobStatusResult == null) { reason = "jobStatusResult is null"; } else { - reason = "server returns non-zero status-code"; + reason = "server returns non-zero status-code. "; + reason += jobStatusResult.getMessage(); } String msg = MessageFormat.format( @@ -224,14 +226,15 @@ public LinkisOperResultAdapter queryJobStatus(String user, String taskID, String "EXE0013", ErrorLevel.ERROR, CommonErrMsg.ExecutionErr, msg, e); } } - Utils.doSleepQuietly(UJESConstants.DRIVER_QUERY_SLEEP_MILLS); + CliUtils.doSleepQuietly(UJESConstants.DRIVER_QUERY_SLEEP_MILLS); } if (jobStatusResult == null || 0 != jobStatusResult.getStatus()) { String reason; if (jobStatusResult == null) { reason = "jobStatusResult is null"; } else { - reason = "server returns non-zero status-code"; + reason = "server returns non-zero status-code. "; + reason += jobStatusResult.getMessage(); } String msg = MessageFormat.format( @@ -255,17 +258,18 @@ private JobInfoResult queryJobInfoInternal(String user, String taskID) while (retryTime++ < MAX_RETRY_TIME) { try { jobInfoResult = client.getJobInfo(executeResult); - logger.debug("job-info: " + Utils.GSON.toJson(jobInfoResult)); + logger.debug("job-info: " + CliUtils.GSON.toJson(jobInfoResult)); if (jobInfoResult == null || 0 != jobInfoResult.getStatus()) { String reason; if (jobInfoResult == null) { reason = "JobInfoResult is null"; } else { - reason = "server returns non-zero status-code"; + reason = "server returns non-zero status-code. "; + reason += jobInfoResult.getMessage(); } String msg = MessageFormat.format( - "Get job info failed. retry time : {0}/{1}. taskID={0}, Reason: {1}", + "Get job info failed. retry time : {0}/{1}. taskID={2}, Reason: {3}", retryTime, MAX_RETRY_TIME, taskID, reason); logger.debug( @@ -288,14 +292,15 @@ private JobInfoResult queryJobInfoInternal(String user, String taskID) "EXE0013", ErrorLevel.ERROR, CommonErrMsg.ExecutionErr, msg, e); } } - Utils.doSleepQuietly(UJESConstants.DRIVER_QUERY_SLEEP_MILLS); + CliUtils.doSleepQuietly(UJESConstants.DRIVER_QUERY_SLEEP_MILLS); } if (jobInfoResult == null || 0 != jobInfoResult.getStatus()) { String reason; if (jobInfoResult == null) { reason = "JobInfoResult is null"; } else { - reason = "server returns non-zero status-code"; + reason = "server returns non-zero status-code. "; + reason += jobInfoResult.getMessage(); } String msg = MessageFormat.format( @@ -321,13 +326,14 @@ public LinkisOperResultAdapter queryRunTimeLogFromLine( while (retryTime++ < MAX_RETRY_TIME) { try { logResult = client.log(jobExecuteResult, fromLine, UJESConstants.MAX_LOG_SIZE); - logger.debug("runtime-log-result:" + Utils.GSON.toJson(logResult)); + logger.debug("runtime-log-result:" + CliUtils.GSON.toJson(logResult)); if (logResult == null || 0 != logResult.getStatus()) { String reason; if (logResult == null) { reason = "JobLogResult is null"; } else { reason = "server returns non-zero status-code"; + reason += logResult.getMessage(); } String msg = MessageFormat.format( @@ -350,14 +356,15 @@ public LinkisOperResultAdapter queryRunTimeLogFromLine( "EXE0016", ErrorLevel.ERROR, CommonErrMsg.ExecutionErr, msg, e); } } - Utils.doSleepQuietly(UJESConstants.DRIVER_QUERY_SLEEP_MILLS); + CliUtils.doSleepQuietly(UJESConstants.DRIVER_QUERY_SLEEP_MILLS); } if (logResult == null || 0 != logResult.getStatus()) { String reason; if (logResult == null) { reason = "JobLogResult is null"; } else { - reason = "server returns non-zero status-code"; + reason = "server returns non-zero status-code. "; + reason += logResult.getMessage(); } String msg = MessageFormat.format( @@ -388,7 +395,7 @@ private OpenLogResult queryPersistedLogInternal(String logPath, String user, Str openLogResult = client.openLog( OpenLogAction.newBuilder().setLogPath(logPath).setProxyUser(user).build()); - logger.debug("persisted-log-result:" + Utils.GSON.toJson(openLogResult)); + logger.debug("persisted-log-result:" + CliUtils.GSON.toJson(openLogResult)); if (openLogResult == null || 0 != openLogResult.getStatus() || StringUtils.isBlank(openLogResult.getLog()[UJESConstants.IDX_FOR_LOG_TYPE_ALL])) { @@ -396,7 +403,8 @@ private OpenLogResult queryPersistedLogInternal(String logPath, String user, Str if (openLogResult == null) { reason = "OpenLogResult is null"; } else if (0 != openLogResult.getStatus()) { - reason = "server returns non-zero status-code"; + reason = "server returns non-zero status-code. "; + reason += openLogResult.getMessage(); } else { reason = "server returns empty log"; } @@ -425,7 +433,7 @@ private OpenLogResult queryPersistedLogInternal(String logPath, String user, Str e); } } - Utils.doSleepQuietly(UJESConstants.DRIVER_QUERY_SLEEP_MILLS); + CliUtils.doSleepQuietly(UJESConstants.DRIVER_QUERY_SLEEP_MILLS); } if (openLogResult == null || 0 != openLogResult.getStatus() @@ -475,7 +483,8 @@ public UJESResultAdapter queryProgress(String user, String taskID, String execId if (jobProgressResult == null) { reason = "JobProgressResult is null"; } else { - reason = "server returns non-zero status-code"; + reason = "server returns non-zero status-code. "; + reason += jobProgressResult.getMessage(); } String msg = MessageFormat.format( @@ -502,7 +511,7 @@ public UJESResultAdapter queryProgress(String user, String taskID, String execId e); } } - Utils.doSleepQuietly(UJESConstants.DRIVER_QUERY_SLEEP_MILLS); + CliUtils.doSleepQuietly(UJESConstants.DRIVER_QUERY_SLEEP_MILLS); } if (jobProgressResult == null || 0 != jobProgressResult.getStatus()) { @@ -510,7 +519,8 @@ public UJESResultAdapter queryProgress(String user, String taskID, String execId if (jobProgressResult == null) { reason = "JobProgressResult is null"; } else { - reason = "server returns non-zero status-code"; + reason = "server returns non-zero status-code. "; + reason += jobProgressResult.getMessage(); } String msg = MessageFormat.format( @@ -588,7 +598,7 @@ public LinkisOperResultAdapter queryResultSetPaths( e); } } - Utils.doSleepQuietly(UJESConstants.DRIVER_QUERY_SLEEP_MILLS); + CliUtils.doSleepQuietly(UJESConstants.DRIVER_QUERY_SLEEP_MILLS); } if (resultSetArray == null || 0 == resultSetArray.length) { String reason; @@ -623,13 +633,14 @@ public LinkisOperResultAdapter queryResultSetGivenResultSetPath( .setPageSize(pageSize) .build(); result = client.resultSet(action); - logger.debug("resultset-result:" + Utils.GSON.toJson(result)); + logger.debug("resultset-result:" + CliUtils.GSON.toJson(result)); if (result == null || 0 != result.getStatus()) { String reason; if (result == null) { reason = "array is null"; } else { - reason = "server returns non-zero status-code"; + reason = "server returns non-zero status-code. "; + reason += result.getMessage(); } String msg = MessageFormat.format( @@ -656,14 +667,15 @@ public LinkisOperResultAdapter queryResultSetGivenResultSetPath( e); } } - Utils.doSleepQuietly(UJESConstants.DRIVER_QUERY_SLEEP_MILLS); + CliUtils.doSleepQuietly(UJESConstants.DRIVER_QUERY_SLEEP_MILLS); } if (result == null || 0 != result.getStatus()) { String reason; if (result == null) { reason = "ResultSetResult is null"; } else { - reason = "server returns non-zero status-code"; + reason = "server returns non-zero status-code. "; + reason += result.getMessage(); } String msg = MessageFormat.format( @@ -691,13 +703,14 @@ public LinkisOperResultAdapter kill(String user, String taskId, String execId) killRequest.setTaskID(taskId); killRequest.setExecID(execId); result = client.kill(killRequest); - logger.debug("job-kill-result:" + Utils.GSON.toJson(result)); + logger.debug("job-kill-result:" + CliUtils.GSON.toJson(result)); if (result == null || 0 != result.getStatus()) { String reason; if (result == null) { reason = "result is null"; } else { - reason = "server returns non-zero status-code"; + reason = "server returns non-zero status-code. "; + reason += result.getMessage(); } String msg = MessageFormat.format( @@ -725,14 +738,15 @@ public LinkisOperResultAdapter kill(String user, String taskId, String execId) e); } } - Utils.doSleepQuietly(UJESConstants.DRIVER_QUERY_SLEEP_MILLS); + CliUtils.doSleepQuietly(UJESConstants.DRIVER_QUERY_SLEEP_MILLS); } if (result == null || 0 != result.getStatus()) { String reason; if (result == null) { reason = "result is null"; } else { - reason = "server returns non-zero status-code"; + reason = "server returns non-zero status-code. "; + reason += result.getMessage(); } String msg = MessageFormat.format( diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/operator/ujes/LinkisOperatorBuilder.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/ujes/LinkisOperBuilder.java similarity index 60% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/operator/ujes/LinkisOperatorBuilder.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/ujes/LinkisOperBuilder.java index 466bb9b7c0..318688c565 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/operator/ujes/LinkisOperatorBuilder.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/ujes/LinkisOperBuilder.java @@ -17,28 +17,23 @@ package org.apache.linkis.cli.application.operator.ujes; -import org.apache.linkis.cli.application.constants.AppKeys; -import org.apache.linkis.cli.core.operator.JobOperatorBuilder; +import org.apache.linkis.cli.application.constants.CliKeys; +import org.apache.linkis.cli.application.entity.context.CliCtx; +import org.apache.linkis.cli.application.entity.operator.JobOper; +import org.apache.linkis.cli.application.operator.JobOperBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class LinkisOperatorBuilder extends JobOperatorBuilder { - private static Logger logger = LoggerFactory.getLogger(LinkisOperatorBuilder.class); +public class LinkisOperBuilder implements JobOperBuilder { + private static Logger logger = LoggerFactory.getLogger(LinkisOperBuilder.class); @Override - public LinkisJobOperator build() { - - ((LinkisJobOperator) targetObj) - .setUJESClient(UJESClientFactory.getReusable(stdVarAccess, sysVarAccess)); - ((LinkisJobOperator) targetObj) - .setServerUrl(stdVarAccess.getVar(String.class, AppKeys.LINKIS_COMMON_GATEWAY_URL)); - - return (LinkisJobOperator) super.build(); - } - - @Override - protected LinkisJobOperator getTargetNewInstance() { - return new LinkisJobOperator(); + public JobOper build(CliCtx ctx) { + LinkisJobOper jobOper = new LinkisJobOper(); + jobOper.setUJESClient(UJESClientFactory.getReusable(ctx.getVarAccess())); + jobOper.setServerUrl( + ctx.getVarAccess().getVar(String.class, CliKeys.LINKIS_COMMON_GATEWAY_URL)); + return jobOper; } } diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/operator/ujes/LinkisOperResultAdapter.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/ujes/LinkisOperResultAdapter.java similarity index 90% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/operator/ujes/LinkisOperResultAdapter.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/ujes/LinkisOperResultAdapter.java index 457a7ba56d..bd22444a2d 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/operator/ujes/LinkisOperResultAdapter.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/ujes/LinkisOperResultAdapter.java @@ -17,8 +17,8 @@ package org.apache.linkis.cli.application.operator.ujes; -import org.apache.linkis.cli.application.interactor.job.data.LinkisResultSet; -import org.apache.linkis.cli.common.entity.job.JobStatus; +import org.apache.linkis.cli.application.entity.job.JobStatus; +import org.apache.linkis.cli.application.interactor.job.common.ResultSet; import java.util.Date; @@ -52,7 +52,7 @@ public interface LinkisOperResultAdapter { Boolean hasNextLogLine(); - LinkisResultSet getResultContent(); + ResultSet getResultContent(); Boolean resultHasNextPage(); diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/operator/ujes/UJESClientContext.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/ujes/UJESClientContext.java similarity index 100% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/operator/ujes/UJESClientContext.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/ujes/UJESClientContext.java diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/operator/ujes/UJESClientFactory.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/ujes/UJESClientFactory.java similarity index 60% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/operator/ujes/UJESClientFactory.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/ujes/UJESClientFactory.java index 2767929e8d..f77a909490 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/operator/ujes/UJESClientFactory.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/ujes/UJESClientFactory.java @@ -17,16 +17,15 @@ package org.apache.linkis.cli.application.operator.ujes; -import org.apache.linkis.cli.application.constants.AppKeys; +import org.apache.linkis.cli.application.constants.CliKeys; import org.apache.linkis.cli.application.constants.LinkisConstants; +import org.apache.linkis.cli.application.entity.var.VarAccess; +import org.apache.linkis.cli.application.exception.BuilderException; +import org.apache.linkis.cli.application.exception.LinkisClientExecutionException; +import org.apache.linkis.cli.application.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.exception.error.ErrorLevel; import org.apache.linkis.cli.application.interactor.validate.UJESContextValidator; -import org.apache.linkis.cli.application.utils.Utils; -import org.apache.linkis.cli.common.entity.validate.Validator; -import org.apache.linkis.cli.common.entity.var.VarAccess; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.exception.BuilderException; -import org.apache.linkis.cli.core.exception.LinkisClientExecutionException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.utils.CliUtils; import org.apache.linkis.httpclient.authentication.AuthenticationStrategy; import org.apache.linkis.httpclient.dws.authentication.StaticAuthenticationStrategy; import org.apache.linkis.httpclient.dws.authentication.TokenAuthenticationStrategy; @@ -47,20 +46,20 @@ public class UJESClientFactory { private static UJESClient client; - public static UJESClient getReusable(VarAccess stdVarAccess, VarAccess sysVarAccess) { + public static UJESClient getReusable(VarAccess stdVarAccess) { if (client == null) { synchronized (UJESClientFactory.class) { if (client == null) { - client = getNew(stdVarAccess, sysVarAccess); + client = getNew(stdVarAccess); } } } return client; } - public static UJESClient getNew(VarAccess stdVarAccess, VarAccess sysVarAccess) { + public static UJESClient getNew(VarAccess stdVarAccess) { try { - DWSClientConfig config = generateDWSClientConfig(stdVarAccess, sysVarAccess); + DWSClientConfig config = generateDWSClientConfig(stdVarAccess); UJESClient ret = new UJESClientImpl(config); logger.info("Linkis ujes client inited."); return ret; @@ -70,9 +69,8 @@ public static UJESClient getNew(VarAccess stdVarAccess, VarAccess sysVarAccess) } } - public static DWSClientConfig generateDWSClientConfig( - VarAccess stdVarAccess, VarAccess sysVarAccess) { - UJESClientContext context = generateContext(stdVarAccess, sysVarAccess); + public static DWSClientConfig generateDWSClientConfig(VarAccess stdVarAccess) { + UJESClientContext context = generateContext(stdVarAccess); try { AuthenticationStrategy authenticationStrategy; if (StringUtils.isBlank(context.getAuthenticationStrategyStr()) @@ -115,53 +113,8 @@ public static DWSClientConfig generateDWSClientConfig( } } - public static DWSClientConfig generateDWSClientConfigForBML( - VarAccess stdVarAccess, VarAccess sysVarAccess) { - UJESClientContext context = generateContext(stdVarAccess, sysVarAccess); - try { - AuthenticationStrategy authenticationStrategy; - if (StringUtils.isBlank(context.getAuthenticationStrategyStr()) - || !LinkisConstants.AUTH_STRATEGY_TOKEN.equalsIgnoreCase( - context.getAuthenticationStrategyStr())) { - authenticationStrategy = - new StaticAuthenticationStrategy(); // this has to be newed here otherwise - // log-in fails for static - } else { - authenticationStrategy = new TokenAuthenticationStrategy(); - } - - DWSClientConfigBuilder builder = DWSClientConfigBuilder.newBuilder(); - DWSClientConfig config = - ((DWSClientConfigBuilder) - (builder - .addServerUrl(context.getGatewayUrl()) - .connectionTimeout(30000) - .discoveryEnabled(false) - .discoveryFrequency(1, TimeUnit.MINUTES) - .loadbalancerEnabled(false) - .maxConnectionSize(5) - .retryEnabled(false) - .readTimeout(context.getReadTimeoutMills()) - .setAuthenticationStrategy(authenticationStrategy) - .setAuthTokenKey("BML-AUTH") - .setAuthTokenValue("BML-AUTH"))) - .setDWSVersion(context.getDwsVersion()) - .build(); - - logger.info("Linkis ujes client inited."); - return config; - } catch (Exception e) { - throw new LinkisClientExecutionException( - "EXE0010", - ErrorLevel.ERROR, - CommonErrMsg.ExecutionInitErr, - "Cannot init DWSClientConfig", - e); - } - } - - private static UJESClientContext generateContext(VarAccess stdVarAccess, VarAccess sysVarAccess) { - String gatewayUrl = stdVarAccess.getVar(String.class, AppKeys.LINKIS_COMMON_GATEWAY_URL); + private static UJESClientContext generateContext(VarAccess stdVarAccess) { + String gatewayUrl = stdVarAccess.getVar(String.class, CliKeys.LINKIS_COMMON_GATEWAY_URL); if (StringUtils.isBlank(gatewayUrl)) { throw new BuilderException( "BLD0007", @@ -170,33 +123,33 @@ private static UJESClientContext generateContext(VarAccess stdVarAccess, VarAcce "Cannot build UjesClientDriverContext: gatewayUrl is empty"); } - String authKey = stdVarAccess.getVar(String.class, AppKeys.LINKIS_COMMON_TOKEN_KEY); - String authValue = stdVarAccess.getVar(String.class, AppKeys.LINKIS_COMMON_TOKEN_VALUE); + String authKey = stdVarAccess.getVar(String.class, CliKeys.LINKIS_COMMON_TOKEN_KEY); + String authValue = stdVarAccess.getVar(String.class, CliKeys.LINKIS_COMMON_TOKEN_VALUE); String authenticationStrategy = stdVarAccess.getVarOrDefault( String.class, - AppKeys.LINKIS_COMMON_AUTHENTICATION_STRATEGY, + CliKeys.LINKIS_COMMON_AUTHENTICATION_STRATEGY, LinkisConstants.AUTH_STRATEGY_STATIC); long connectionTimeout = stdVarAccess.getVarOrDefault( - Long.class, AppKeys.UJESCLIENT_COMMON_CONNECTT_TIMEOUT, 30000L); + Long.class, CliKeys.UJESCLIENT_COMMON_CONNECTT_TIMEOUT, 30000L); boolean discoveryEnabled = stdVarAccess.getVarOrDefault( - Boolean.class, AppKeys.UJESCLIENT_COMMON_DISCOVERY_ENABLED, false); + Boolean.class, CliKeys.UJESCLIENT_COMMON_DISCOVERY_ENABLED, false); boolean loadBalancerEnabled = stdVarAccess.getVarOrDefault( - Boolean.class, AppKeys.UJESCLIENT_COMMON_LOADBALANCER_ENABLED, true); + Boolean.class, CliKeys.UJESCLIENT_COMMON_LOADBALANCER_ENABLED, true); int maxConnectionSize = stdVarAccess.getVarOrDefault( - Integer.class, AppKeys.UJESCLIENT_COMMON_MAX_CONNECTION_SIZE, 5); + Integer.class, CliKeys.UJESCLIENT_COMMON_MAX_CONNECTION_SIZE, 5); boolean retryEnabled = - stdVarAccess.getVarOrDefault(Boolean.class, AppKeys.UJESCLIENT_COMMON_RETRY_ENABLED, false); + stdVarAccess.getVarOrDefault(Boolean.class, CliKeys.UJESCLIENT_COMMON_RETRY_ENABLED, false); long readTimeout = - stdVarAccess.getVarOrDefault(Long.class, AppKeys.UJESCLIENT_COMMON_READTIMEOUT, 30000L); + stdVarAccess.getVarOrDefault(Long.class, CliKeys.UJESCLIENT_COMMON_READTIMEOUT, 30000L); String dwsVersion = - stdVarAccess.getVarOrDefault(String.class, AppKeys.UJESCLIENT_COMMON_DWS_VERSION, "v1"); + stdVarAccess.getVarOrDefault(String.class, CliKeys.UJESCLIENT_COMMON_DWS_VERSION, "v1"); UJESClientContext context = new UJESClientContext(); @@ -212,8 +165,8 @@ private static UJESClientContext generateContext(VarAccess stdVarAccess, VarAcce context.setReadTimeoutMills(readTimeout); context.setDwsVersion(dwsVersion); - logger.info("==========UJES_CTX============\n" + Utils.GSON.toJson(context)); - Validator ctxValidator = new UJESContextValidator(); + logger.info("==========UJES_CTX============\n" + CliUtils.GSON.toJson(context)); + UJESContextValidator ctxValidator = new UJESContextValidator(); ctxValidator.doValidation(context); return context; } diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/operator/ujes/UJESConstants.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/ujes/UJESConstants.java similarity index 93% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/operator/ujes/UJESConstants.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/ujes/UJESConstants.java index 845949079f..6166f7b2fe 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/operator/ujes/UJESConstants.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/ujes/UJESConstants.java @@ -40,7 +40,4 @@ public class UJESConstants { public static final Integer IDX_FOR_LOG_TYPE_ALL = 3; // 0: Error 1: WARN 2:INFO 3: ALL public static final int DEFAULT_PAGE_SIZE = 500; - - public static final String DEFAULT_SPARK_ENGINE = "spark-2.4.3"; - public static final String DEFAULT_HIVE_ENGINE = "hive-1.2.1"; } diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/operator/ujes/UJESResultAdapter.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/ujes/UJESResultAdapter.java similarity index 90% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/operator/ujes/UJESResultAdapter.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/ujes/UJESResultAdapter.java index d402e01fe8..6c746ff57c 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/operator/ujes/UJESResultAdapter.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/ujes/UJESResultAdapter.java @@ -18,17 +18,16 @@ package org.apache.linkis.cli.application.operator.ujes; import org.apache.linkis.cli.application.constants.LinkisKeys; -import org.apache.linkis.cli.application.interactor.job.LinkisJobStatus; -import org.apache.linkis.cli.application.interactor.job.data.LinkisResultSet; -import org.apache.linkis.cli.application.operator.OperatorUtils; +import org.apache.linkis.cli.application.entity.job.JobStatus; +import org.apache.linkis.cli.application.exception.TransformerException; +import org.apache.linkis.cli.application.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.exception.error.ErrorLevel; +import org.apache.linkis.cli.application.interactor.job.common.LinkisJobStatus; +import org.apache.linkis.cli.application.interactor.job.common.ResultSet; import org.apache.linkis.cli.application.operator.ujes.result.OpenLogResult2; import org.apache.linkis.cli.application.operator.ujes.result.ResultSetResult2; import org.apache.linkis.cli.application.operator.ujes.result.UJESResult; -import org.apache.linkis.cli.application.utils.Utils; -import org.apache.linkis.cli.common.entity.job.JobStatus; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.exception.TransformerException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.utils.CliUtils; import org.apache.linkis.httpclient.dws.response.DWSResult; import org.apache.linkis.ujes.client.request.UserAction; import org.apache.linkis.ujes.client.response.JobInfoResult; @@ -170,7 +169,7 @@ public String getStrongerExecId() { execId = (String) ((JobInfoResult) result).getTask().get(LinkisKeys.KEY_STRONGER_EXECID); } } - if (Utils.isValidExecId(execId)) { + if (CliUtils.isValidExecId(execId)) { return execId; } return null; @@ -277,8 +276,7 @@ public String getLog() { ((OpenLogResult2) result).getResult().getLog()[UJESConstants.IDX_FOR_LOG_TYPE_ALL]; Integer fromLine = ((OpenLogResult2) result).getFromLine(); return StringUtils.substring( - allLog, - OperatorUtils.getFirstIndexSkippingLines(allLog, fromLine == null ? 0 : fromLine)); + allLog, getFirstIndexSkippingLines(allLog, fromLine == null ? 0 : fromLine)); } return null; } @@ -294,7 +292,7 @@ public Integer getNextLogLine() { if (result instanceof OpenLogResult2 && ((OpenLogResult2) result).getResult() != null && ((OpenLogResult2) result).getResult().getLog() != null) { - return OperatorUtils.getNumOfLines( + return getNumOfLines( ((OpenLogResult2) result).getResult().getLog()[UJESConstants.IDX_FOR_LOG_TYPE_ALL]); } return null; @@ -315,7 +313,7 @@ public Boolean hasNextLogLine() { } @Override - public LinkisResultSet getResultContent() { + public ResultSet getResultContent() { if (result == null) { return null; } @@ -323,7 +321,7 @@ public LinkisResultSet getResultContent() { && ((ResultSetResult2) result).getResultSetResult() != null && ((ResultSetResult2) result).getResultSetResult().getFileContent() != null && ((ResultSetResult2) result).getResultSetResult().getMetadata() != null) { - LinkisResultSet ret = new LinkisResultSet(); + ResultSet ret = new ResultSet(); ret.setResultsetIdx(((ResultSetResult2) result).getIdxResultSet()); if (((ResultSetResult2) result).getResultSetResult().getMetadata() != null) { ret.setResultMeta( @@ -560,4 +558,54 @@ public String getRequestApplicationName() { } return null; } + + private int getNumOfLines(String str) { + if (str == null || str.length() == 0) { + return 0; + } + int lines = 1; + int len = str.length(); + for (int pos = 0; pos < len; pos++) { + char c = str.charAt(pos); + if (c == '\r') { + lines++; + if (pos + 1 < len && str.charAt(pos + 1) == '\n') { + pos++; + } + } else if (c == '\n') { + lines++; + } + } + return lines; + } + + private int getFirstIndexSkippingLines(String str, Integer lines) { + if (str == null || str.length() == 0 || lines < 0) { + return -1; + } + if (lines == 0) { + return 0; + } + + int curLineIdx = 0; + int len = str.length(); + for (int pos = 0; pos < len; pos++) { + char c = str.charAt(pos); + if (c == '\r') { + curLineIdx++; + if (pos + 1 < len && str.charAt(pos + 1) == '\n') { + pos++; + } + } else if (c == '\n') { + curLineIdx++; + } else { + continue; + } + + if (curLineIdx >= lines) { + return pos + 1; + } + } + return -1; + } } diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/operator/ujes/result/OpenLogResult2.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/ujes/result/OpenLogResult2.java similarity index 100% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/operator/ujes/result/OpenLogResult2.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/ujes/result/OpenLogResult2.java diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/operator/ujes/result/ResultSetResult2.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/ujes/result/ResultSetResult2.java similarity index 100% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/operator/ujes/result/ResultSetResult2.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/ujes/result/ResultSetResult2.java diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/operator/ujes/result/UJESResult.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/ujes/result/UJESResult.java similarity index 100% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/operator/ujes/result/UJESResult.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/ujes/result/UJESResult.java diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/present/HelpInfoPresenter.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/present/HelpPresenter.java similarity index 65% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/present/HelpInfoPresenter.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/present/HelpPresenter.java index 186081a560..f02be7fefc 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/present/HelpInfoPresenter.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/present/HelpPresenter.java @@ -15,23 +15,20 @@ * limitations under the License. */ -package org.apache.linkis.cli.core.present; +package org.apache.linkis.cli.application.present; -import org.apache.linkis.cli.common.entity.command.CmdOption; -import org.apache.linkis.cli.common.entity.command.CmdTemplate; -import org.apache.linkis.cli.common.entity.present.Model; -import org.apache.linkis.cli.common.entity.present.PresentWay; -import org.apache.linkis.cli.common.entity.present.Presenter; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.exception.PresenterException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; -import org.apache.linkis.cli.core.interactor.command.template.option.MapOption; -import org.apache.linkis.cli.core.interactor.command.template.option.Parameter; -import org.apache.linkis.cli.core.interactor.command.template.option.StdOption; -import org.apache.linkis.cli.core.present.display.DisplayOperator; -import org.apache.linkis.cli.core.present.display.StdOutWriter; -import org.apache.linkis.cli.core.present.display.data.StdoutDisplayData; -import org.apache.linkis.cli.core.present.model.HelpInfoModel; +import org.apache.linkis.cli.application.entity.command.CmdOption; +import org.apache.linkis.cli.application.entity.command.CmdTemplate; +import org.apache.linkis.cli.application.entity.present.Model; +import org.apache.linkis.cli.application.entity.present.Presenter; +import org.apache.linkis.cli.application.exception.PresenterException; +import org.apache.linkis.cli.application.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.exception.error.ErrorLevel; +import org.apache.linkis.cli.application.interactor.command.template.option.MapOption; +import org.apache.linkis.cli.application.interactor.command.template.option.Parameter; +import org.apache.linkis.cli.application.interactor.command.template.option.StdOption; +import org.apache.linkis.cli.application.present.model.HelpInfoModel; +import org.apache.linkis.cli.application.utils.LoggerManager; import java.util.ArrayList; import java.util.List; @@ -39,12 +36,11 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class HelpInfoPresenter implements Presenter { - private static Logger logger = LoggerFactory.getLogger(HelpInfoPresenter.class); - DisplayOperator driver = new StdOutWriter(); +public class HelpPresenter implements Presenter { + private static Logger logger = LoggerFactory.getLogger(HelpPresenter.class); @Override - public void present(Model model, PresentWay presentWay) { + public void present(Model model) { if (!(model instanceof HelpInfoModel)) { throw new PresenterException( "PST0010", @@ -52,15 +48,12 @@ public void present(Model model, PresentWay presentWay) { CommonErrMsg.PresenterErr, "Input for HelpInfoPresenter is not instance of model"); } - if (driver == null) { - throw new PresenterException( - "PST0007", ErrorLevel.ERROR, CommonErrMsg.PresenterErr, "Driver is null"); - } + HelpInfoModel helpInfoModel = (HelpInfoModel) model; String helpInfo = getUsage(helpInfoModel.getTemplate()); - driver.doOutput(new StdoutDisplayData(helpInfo)); + LoggerManager.getPlaintTextLogger().info(helpInfo); } /** Help info for sub-command */ diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/validate/PropsValidator.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/present/JobInfoPresenter.java similarity index 63% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/validate/PropsValidator.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/present/JobInfoPresenter.java index 6231f4eb88..3a264f827c 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/validate/PropsValidator.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/present/JobInfoPresenter.java @@ -15,17 +15,16 @@ * limitations under the License. */ -package org.apache.linkis.cli.core.interactor.validate; +package org.apache.linkis.cli.application.present; -import org.apache.linkis.cli.common.entity.properties.ClientProperties; -import org.apache.linkis.cli.common.entity.validate.Validator; -import org.apache.linkis.cli.common.exception.LinkisClientRuntimeException; +import org.apache.linkis.cli.application.entity.present.Model; +import org.apache.linkis.cli.application.entity.present.Presenter; +import org.apache.linkis.cli.application.utils.CliUtils; +import org.apache.linkis.cli.application.utils.LoggerManager; -public class PropsValidator implements Validator { +public class JobInfoPresenter implements Presenter { @Override - public void doValidation(Object input) throws LinkisClientRuntimeException { - if (!(input instanceof ClientProperties)) { - // TODO:throw - } + public void present(Model model) { + LoggerManager.getPlaintTextLogger().info(CliUtils.GSON.toJson(model)); } } diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/present/LinkisLogPresenter.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/present/LogPresenter.java similarity index 61% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/present/LinkisLogPresenter.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/present/LogPresenter.java index 291d7314db..9710be95db 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/present/LinkisLogPresenter.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/present/LogPresenter.java @@ -17,31 +17,34 @@ package org.apache.linkis.cli.application.present; +import org.apache.linkis.cli.application.entity.present.Model; +import org.apache.linkis.cli.application.entity.present.Presenter; +import org.apache.linkis.cli.application.exception.PresenterException; +import org.apache.linkis.cli.application.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.exception.error.ErrorLevel; import org.apache.linkis.cli.application.observer.event.LinkisClientEvent; import org.apache.linkis.cli.application.observer.listener.LinkisClientListener; import org.apache.linkis.cli.application.present.model.LinkisLogModel; -import org.apache.linkis.cli.common.entity.present.Model; -import org.apache.linkis.cli.common.entity.present.PresentWay; -import org.apache.linkis.cli.common.entity.present.Presenter; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.exception.PresenterException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; -import org.apache.linkis.cli.core.present.PresentModeImpl; -import org.apache.linkis.cli.core.present.display.DisplayOperFactory; -import org.apache.linkis.cli.core.present.display.DisplayOperator; -import org.apache.linkis.cli.core.present.display.data.StdoutDisplayData; -import org.apache.linkis.cli.core.utils.CommonUtils; +import org.apache.linkis.cli.application.utils.CliUtils; +import org.apache.linkis.cli.application.utils.LoggerManager; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class LinkisLogPresenter implements Presenter, LinkisClientListener { - private static Logger logger = LoggerFactory.getLogger(LinkisLogPresenter.class); +public class LogPresenter implements Presenter, LinkisClientListener { + private static Logger logger = LoggerFactory.getLogger(LogPresenter.class); @Override - public void present(Model model, PresentWay presentWay) { + public void update(LinkisClientEvent event, Object msg) { + Model model = new LinkisLogModel(); + model.buildModel(msg); + this.present(model); + } + + @Override + public void present(Model model) { if (!(model instanceof LinkisLogModel)) { throw new PresenterException( "PST0001", @@ -50,24 +53,16 @@ public void present(Model model, PresentWay presentWay) { "Input model for \"LinkisLogPresenter\" is not instance of \"LinkisJobIncLogModel\""); } LinkisLogModel logModel = (LinkisLogModel) model; - DisplayOperator displayOper = DisplayOperFactory.getDisplayOper(PresentModeImpl.STDOUT); while (!logModel.logFinReceived()) { String incLog = logModel.consumeLog(); if (StringUtils.isNotEmpty(incLog)) { - displayOper.doOutput(new StdoutDisplayData(incLog)); + LoggerManager.getPlaintTextLogger().info(incLog); } - CommonUtils.doSleepQuietly(500l); + CliUtils.doSleepQuietly(500l); } String incLog = logModel.consumeLog(); if (StringUtils.isNotEmpty(incLog)) { - displayOper.doOutput(new StdoutDisplayData(incLog)); + LoggerManager.getPlaintTextLogger().info(incLog); } } - - @Override - public void update(LinkisClientEvent event, Object msg) { - Model model = new LinkisLogModel(); - model.buildModel(msg); - this.present(model, null); - } } diff --git a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/present/ResultPresenter.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/present/ResultPresenter.java new file mode 100644 index 0000000000..3c7281a7eb --- /dev/null +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/present/ResultPresenter.java @@ -0,0 +1,301 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.cli.application.present; + +import org.apache.linkis.cli.application.constants.CliConstants; +import org.apache.linkis.cli.application.entity.present.Model; +import org.apache.linkis.cli.application.entity.present.Presenter; +import org.apache.linkis.cli.application.exception.PresenterException; +import org.apache.linkis.cli.application.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.exception.error.ErrorLevel; +import org.apache.linkis.cli.application.interactor.job.common.ResultSet; +import org.apache.linkis.cli.application.observer.event.LinkisClientEvent; +import org.apache.linkis.cli.application.observer.listener.LinkisClientListener; +import org.apache.linkis.cli.application.present.file.ResultFileWriter; +import org.apache.linkis.cli.application.present.model.LinkisResultModel; +import org.apache.linkis.cli.application.utils.CliUtils; +import org.apache.linkis.cli.application.utils.LoggerManager; + +import org.apache.commons.lang3.StringUtils; + +import java.text.MessageFormat; +import java.util.*; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ResultPresenter implements Presenter, LinkisClientListener { + private static Logger logger = LoggerFactory.getLogger(ResultPresenter.class); + private Boolean writeToFile = false; + private String filePath = ""; + + public ResultPresenter() {} + + public ResultPresenter(Boolean writeToFile, String filePath) { + this.writeToFile = writeToFile; + this.filePath = filePath; + } + + @Override + public void update(LinkisClientEvent event, Object msg) { + Model model = new LinkisResultModel(); + model.buildModel(msg); + this.present(model); + } + + @Override + public void present(Model model) { + if (!(model instanceof LinkisResultModel)) { + throw new PresenterException( + "PST0001", + ErrorLevel.ERROR, + CommonErrMsg.PresenterErr, + "Input model for \"LinkisResultPresenter\" is not instance of \"LinkisResultModel\""); + } + LinkisResultModel resultModel = (LinkisResultModel) model; + + LoggerManager.getPlaintTextLogger().info(formatResultIndicator(resultModel)); + + if (!resultModel.getJobStatus().isJobSuccess()) { + LoggerManager.getInformationLogger() + .info("JobStatus is not \'success\'. Will not retrieve result-set."); + return; + } + String msg = ""; + if (resultModel.hasResult()) { + msg = + "Retrieving result-set, may take time if result-set is large, please do not exit program."; + } else { + msg = "Your job has no result."; + } + LoggerManager.getInformationLogger().info(msg); + + int preIdx = -1; + StringBuilder resultSb = new StringBuilder(); + + while (!resultModel.resultFinReceived()) { + preIdx = presentOneIteration(resultModel, preIdx, resultSb); + CliUtils.doSleepQuietly(500l); + } + presentOneIteration(resultModel, preIdx, resultSb); + + if (writeToFile) { + LoggerManager.getInformationLogger() + .info("ResultSet has been successfully written to path: " + filePath); + } + } + + protected int presentOneIteration( + LinkisResultModel resultModel, int preIdx, StringBuilder resultSb) { + List resultSets = resultModel.consumeResultContent(); + if (resultSets != null && !resultSets.isEmpty()) { + for (ResultSet c : resultSets) { + int idxResultset = c.getResultsetIdx(); + /** + * Notice: we assume result-sets are visited one by one in non-descending order!!! i.e. + * either idxResultset == preIdx or idxResultset - preIdx == 1 i.e. resultsets[0] -> + * resultsets[1] -> ... + */ + if (idxResultset - preIdx != 0 && idxResultset - preIdx != 1) { + throw new PresenterException( + "PST0002", + ErrorLevel.ERROR, + CommonErrMsg.PresenterErr, + "Linkis resultsets are visited in descending order or are not visited one-by-one"); + } + + boolean flag = idxResultset > preIdx; + if (idxResultset - preIdx == 1) { + resultSb.setLength(0); + resultSb + .append(MessageFormat.format(CliConstants.RESULTSET_LOGO, idxResultset + 1)) + .append(System.lineSeparator()); + if (c.getResultMeta() != null) { + resultSb.append(CliConstants.RESULTSET_META_BEGIN_LOGO).append(System.lineSeparator()); + resultSb.append(formatResultMeta(c.getResultMeta())); + resultSb.append(CliConstants.RESULTSET_META_END_LOGO).append(System.lineSeparator()); + } + } + preIdx = idxResultset; + String contentStr = formatResultContent(c.getResultMeta(), c.getContent()); + if (contentStr != null) { + resultSb.append(contentStr); + } + if (resultSb.length() != 0) { + if (writeToFile) { + String resultFileName = + resultModel.getUser() + + "-task-" + + resultModel.getJobID() + + "-result-" + + String.valueOf(idxResultset + 1) + + ".txt"; + ResultFileWriter.writeToFile(filePath, resultFileName, resultSb.toString(), flag); + } else { + LoggerManager.getPlaintTextLogger().info(resultSb.toString()); + } + resultSb.setLength(0); + } + } + } + return preIdx; + } + + protected String formatResultMeta(List> metaData) { + + StringBuilder outputBuilder = new StringBuilder(); + + if (metaData == null || metaData.size() == 0) { + return null; + } + + List titles = new ArrayList<>(); + + // gather keys as title + for (LinkedHashMap mapElement : metaData) { + if (mapElement == null || mapElement.size() == 0) { + continue; + } + + Set> entrySet = mapElement.entrySet(); + if (entrySet == null) { + break; + } + for (Map.Entry entry : entrySet) { + String key = entry.getKey(); + if (key != null && !titles.contains(key)) { + titles.add(key); + outputBuilder.append(key).append("\t"); + } + } + } + + outputBuilder.append(System.lineSeparator()); + + // gather value and print to output + for (LinkedHashMap mapElement : metaData) { + if (mapElement == null || mapElement.size() == 0) { + continue; + } + String candidate; + for (String title : titles) { + if (mapElement.containsKey(title)) { + candidate = mapElement.get(title); + } else { + candidate = "NULL"; + } + outputBuilder.append(candidate).append("\t"); + } + outputBuilder.append(System.lineSeparator()); + } + return outputBuilder.toString(); + } + + protected String formatResultContent( + List> metaData, List> contentData) { + + StringBuilder outputBuilder = new StringBuilder(); + if (contentData == null || contentData.size() == 0) { // finished + return null; + } + + int listLen = contentData.size(); + for (int i = 0; i < listLen; i++) { + List listElement = contentData.get(i); + if (listElement == null || listElement.size() == 0) { + continue; + } + for (String element : listElement) { + outputBuilder.append(element).append("\t"); + } + if (i < listLen - 1) { + outputBuilder.append(System.lineSeparator()); + } + } + + return outputBuilder.toString(); + } + + protected String formatResultIndicator(LinkisResultModel model) { + StringBuilder infoBuilder = new StringBuilder(); + String extraMsgStr = ""; + + if (model.getExtraMessage() != null) { + extraMsgStr = model.getExtraMessage(); + } + if (model.getJobStatus().isJobSuccess()) { + + LoggerManager.getInformationLogger() + .info("Job execute successfully! Will try get execute result"); + infoBuilder + .append("============Result:================") + .append(System.lineSeparator()) + .append("TaskId:") + .append(model.getJobID()) + .append(System.lineSeparator()) + .append("ExecId: ") + .append(model.getExecID()) + .append(System.lineSeparator()) + .append("User:") + .append(model.getUser()) + .append(System.lineSeparator()) + .append("Current job status:") + .append(model.getJobStatus()) + .append(System.lineSeparator()) + .append("extraMsg: ") + .append(extraMsgStr) + .append(System.lineSeparator()) + .append("result: ") + .append(extraMsgStr) + .append(System.lineSeparator()); + } else if (model.getJobStatus().isJobFinishedState()) { + LoggerManager.getInformationLogger().info("Job failed! Will not try get execute result."); + infoBuilder + .append("============Result:================") + .append(System.lineSeparator()) + .append("TaskId:") + .append(model.getJobID()) + .append(System.lineSeparator()) + .append("ExecId: ") + .append(model.getExecID()) + .append(System.lineSeparator()) + .append("User:") + .append(model.getUser()) + .append(System.lineSeparator()) + .append("Current job status:") + .append(model.getJobStatus()) + .append(System.lineSeparator()) + .append("extraMsg: ") + .append(extraMsgStr) + .append(System.lineSeparator()); + if (model.getErrCode() != null) { + infoBuilder.append("errCode: ").append(model.getErrCode()).append(System.lineSeparator()); + } + if (StringUtils.isNotBlank(model.getErrDesc())) { + infoBuilder.append("errDesc: ").append(model.getErrDesc()).append(System.lineSeparator()); + } + } else { + throw new PresenterException( + "PST0011", + ErrorLevel.ERROR, + CommonErrMsg.PresenterErr, + "Job is not completed but triggered ResultPresenter"); + } + return infoBuilder.toString(); + } +} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/present/display/PlainTextFileWriter.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/present/file/ResultFileWriter.java similarity index 71% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/present/display/PlainTextFileWriter.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/present/file/ResultFileWriter.java index 789b344501..c2d47e2b7a 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/present/display/PlainTextFileWriter.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/present/file/ResultFileWriter.java @@ -15,37 +15,23 @@ * limitations under the License. */ -package org.apache.linkis.cli.core.present.display; +package org.apache.linkis.cli.application.present.file; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.exception.PresenterException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; -import org.apache.linkis.cli.core.present.display.data.DisplayData; -import org.apache.linkis.cli.core.present.display.data.FileDisplayData; +import org.apache.linkis.cli.application.exception.PresenterException; +import org.apache.linkis.cli.application.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.exception.error.ErrorLevel; import java.io.BufferedWriter; import java.io.File; import java.io.FileOutputStream; import java.io.OutputStreamWriter; -public class PlainTextFileWriter implements DisplayOperator { - @Override - public void doOutput(DisplayData data) { - if (!(data instanceof FileDisplayData)) { - throw new PresenterException( - "PST0004", - ErrorLevel.ERROR, - CommonErrMsg.PresentDriverErr, - "input data is not instance of FileDisplayData"); - } +public class ResultFileWriter { - String pathName = ((FileDisplayData) data).getPathName(); - String fileName = pathName + File.separator + ((FileDisplayData) data).getFileName(); - String content = ((FileDisplayData) data).getContent(); - Boolean overWrite = ((FileDisplayData) data).getCreateNewFile(); + public static void writeToFile( + String pathName, String fileName, String content, Boolean overWrite) { File dir = new File(pathName); - File file = new File(fileName); if (!dir.exists()) { try { @@ -60,6 +46,8 @@ public void doOutput(DisplayData data) { } } + File file = new File(dir.getAbsolutePath() + File.separator + fileName); + if (overWrite || !file.exists()) { try { file.createNewFile(); diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/present/model/HelpInfoModel.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/present/model/HelpInfoModel.java similarity index 77% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/present/model/HelpInfoModel.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/present/model/HelpInfoModel.java index c2b5f5e772..46dcec2d5e 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/present/model/HelpInfoModel.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/present/model/HelpInfoModel.java @@ -15,13 +15,13 @@ * limitations under the License. */ -package org.apache.linkis.cli.core.present.model; +package org.apache.linkis.cli.application.present.model; -import org.apache.linkis.cli.common.entity.command.CmdTemplate; -import org.apache.linkis.cli.common.entity.present.Model; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.exception.TransformerException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.entity.command.CmdTemplate; +import org.apache.linkis.cli.application.entity.present.Model; +import org.apache.linkis.cli.application.exception.TransformerException; +import org.apache.linkis.cli.application.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.exception.error.ErrorLevel; public class HelpInfoModel implements Model { CmdTemplate template; diff --git a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/present/model/LinkisJobInfoModel.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/present/model/LinkisJobInfoModel.java new file mode 100644 index 0000000000..b428a788dd --- /dev/null +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/present/model/LinkisJobInfoModel.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.cli.application.present.model; + +import org.apache.linkis.cli.application.entity.job.JobStatus; +import org.apache.linkis.cli.application.entity.present.Model; +import org.apache.linkis.cli.application.exception.TransformerException; +import org.apache.linkis.cli.application.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.exception.error.ErrorLevel; +import org.apache.linkis.cli.application.operator.ujes.LinkisOperResultAdapter; + +import java.util.Date; + +public class LinkisJobInfoModel implements Model { + + private String cid; + private String jobId; + private String message; + private String exception; + private String cause; + + private String taskID; + private String instance; + private String simpleExecId; + private String execId; + private String umUser; + private String executionCode; + private String logPath; + private JobStatus status; + private String engineType; + private String runType; + private Long costTime; + private Date createdTime; + private Date updatedTime; + private Date engineStartTime; + private Integer errCode; + private String errMsg; + private String executeApplicationName; + private String requestApplicationName; + private Float progress; + + @Override + public void buildModel(Object data) { + if (!(data instanceof LinkisOperResultAdapter)) { + throw new TransformerException( + "TFM0010", + ErrorLevel.ERROR, + CommonErrMsg.TransformerException, + "Failed to init LinkisJobInfoModel: " + + data.getClass().getCanonicalName() + + "is not instance of \"LinkisOperResultAdapter\""); + } + this.jobId = ((LinkisOperResultAdapter) data).getJobID(); + this.taskID = ((LinkisOperResultAdapter) data).getJobID(); + this.instance = ((LinkisOperResultAdapter) data).getInstance(); + this.simpleExecId = ((LinkisOperResultAdapter) data).getSimpleExecId(); + this.execId = ((LinkisOperResultAdapter) data).getStrongerExecId(); + this.umUser = ((LinkisOperResultAdapter) data).getUmUser(); + this.executionCode = ((LinkisOperResultAdapter) data).getExecutionCode(); + this.logPath = ((LinkisOperResultAdapter) data).getLogPath(); + this.status = ((LinkisOperResultAdapter) data).getJobStatus(); + this.engineType = ((LinkisOperResultAdapter) data).getEngineType(); + this.runType = ((LinkisOperResultAdapter) data).getRunType(); + this.costTime = ((LinkisOperResultAdapter) data).getCostTime(); + this.createdTime = ((LinkisOperResultAdapter) data).getCreatedTime(); + this.updatedTime = ((LinkisOperResultAdapter) data).getUpdatedTime(); + this.engineStartTime = ((LinkisOperResultAdapter) data).getEngineStartTime(); + this.errCode = ((LinkisOperResultAdapter) data).getErrCode(); + this.errMsg = ((LinkisOperResultAdapter) data).getErrDesc(); + this.executeApplicationName = ((LinkisOperResultAdapter) data).getExecuteApplicationName(); + this.requestApplicationName = ((LinkisOperResultAdapter) data).getRequestApplicationName(); + this.progress = ((LinkisOperResultAdapter) data).getJobProgress(); + } +} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/present/model/LinkisLogModel.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/present/model/LinkisLogModel.java similarity index 74% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/present/model/LinkisLogModel.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/present/model/LinkisLogModel.java index 503c778339..e580547887 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/present/model/LinkisLogModel.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/present/model/LinkisLogModel.java @@ -17,18 +17,18 @@ package org.apache.linkis.cli.application.present.model; -import org.apache.linkis.cli.application.interactor.job.data.LinkisLogData; -import org.apache.linkis.cli.common.entity.present.Model; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.exception.TransformerException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.entity.present.Model; +import org.apache.linkis.cli.application.exception.TransformerException; +import org.apache.linkis.cli.application.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.exception.error.ErrorLevel; +import org.apache.linkis.cli.application.interactor.job.common.LogData; public class LinkisLogModel implements Model { - private LinkisLogData data; + private LogData data; @Override public void buildModel(Object data) { - if (!(data instanceof LinkisLogData)) { + if (!(data instanceof LogData)) { throw new TransformerException( "TFM0010", ErrorLevel.ERROR, @@ -37,7 +37,7 @@ public void buildModel(Object data) { + data.getClass().getCanonicalName() + "is not instance of \"LinkisLogData\""); } - this.data = (LinkisLogData) data; + this.data = (LogData) data; } public String consumeLog() { @@ -45,6 +45,6 @@ public String consumeLog() { } public boolean logFinReceived() { - return data.logFinReceived(); + return data.isLogFin(); } } diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/present/model/LinkisResultModel.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/present/model/LinkisResultModel.java similarity index 63% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/present/model/LinkisResultModel.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/present/model/LinkisResultModel.java index 1dce057c3e..fd9e8eb9a2 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/present/model/LinkisResultModel.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/present/model/LinkisResultModel.java @@ -17,22 +17,22 @@ package org.apache.linkis.cli.application.present.model; -import org.apache.linkis.cli.application.interactor.job.data.LinkisResultData; -import org.apache.linkis.cli.application.interactor.job.data.LinkisResultSet; -import org.apache.linkis.cli.common.entity.job.JobStatus; -import org.apache.linkis.cli.common.entity.present.Model; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.exception.TransformerException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.entity.job.JobStatus; +import org.apache.linkis.cli.application.entity.present.Model; +import org.apache.linkis.cli.application.exception.TransformerException; +import org.apache.linkis.cli.application.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.exception.error.ErrorLevel; +import org.apache.linkis.cli.application.interactor.job.common.ResultData; +import org.apache.linkis.cli.application.interactor.job.common.ResultSet; import java.util.List; public class LinkisResultModel implements Model { - private LinkisResultData data; + private ResultData data; @Override public void buildModel(Object data) { - if (!(data instanceof LinkisResultData)) { + if (!(data instanceof ResultData)) { throw new TransformerException( "TFM0010", ErrorLevel.ERROR, @@ -41,15 +41,15 @@ public void buildModel(Object data) { + data.getClass().getCanonicalName() + "is not instance of \"LinkisResultData\""); } - this.data = (LinkisResultData) data; + this.data = (ResultData) data; } - public List consumeResultContent() { + public List consumeResultContent() { return data.consumeResultContent(); } public boolean resultFinReceived() { - return data.resultFinReceived(); + return data.isResultFin(); } public JobStatus getJobStatus() { @@ -67,4 +67,20 @@ public String getUser() { public boolean hasResult() { return data.hasResult(); } + + public String getExecID() { + return data.getExecID(); + } + + public Integer getErrCode() { + return data.getErrCode(); + } + + public String getErrDesc() { + return data.getErrDesc(); + } + + public String getExtraMessage() { + return data.getExtraMessage(); + } } diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/utils/ExecutionUtils.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/utils/CliUtils.java similarity index 65% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/utils/ExecutionUtils.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/utils/CliUtils.java index 82a3d8bff2..3f8d86d48e 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/utils/ExecutionUtils.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/utils/CliUtils.java @@ -17,31 +17,38 @@ package org.apache.linkis.cli.application.utils; -import org.apache.linkis.cli.application.constants.AppKeys; +import org.apache.linkis.cli.application.constants.CliKeys; import org.apache.linkis.cli.application.constants.LinkisConstants; -import org.apache.linkis.cli.common.entity.var.VarAccess; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.exception.BuilderException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; -import org.apache.linkis.cli.core.utils.LogUtils; +import org.apache.linkis.cli.application.entity.var.VarAccess; +import org.apache.linkis.cli.application.exception.BuilderException; +import org.apache.linkis.cli.application.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.exception.error.ErrorLevel; import org.apache.commons.lang3.StringUtils; import java.io.*; +import java.util.HashMap; +import java.util.Map; import java.util.Set; -public class ExecutionUtils { +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; + +public class CliUtils { + public static final Gson GSON = + new GsonBuilder().setPrettyPrinting().disableHtmlEscaping().create(); + public static String getSubmitUser(VarAccess stdVarAccess, String osUser, Set adminSet) { String enableSpecifyUserStr = stdVarAccess.getVar( - String.class, AppKeys.LINKIS_CLIENT_NONCUSTOMIZABLE_ENABLE_USER_SPECIFICATION); + String.class, CliKeys.LINKIS_CLIENT_NONCUSTOMIZABLE_ENABLE_USER_SPECIFICATION); Boolean enableSpecifyUser = Boolean.parseBoolean(enableSpecifyUserStr) || adminSet.contains(osUser); String authenticationStrategy = stdVarAccess.getVarOrDefault( String.class, - AppKeys.LINKIS_COMMON_AUTHENTICATION_STRATEGY, + CliKeys.LINKIS_COMMON_AUTHENTICATION_STRATEGY, LinkisConstants.AUTH_STRATEGY_STATIC); String submitUsr; @@ -53,8 +60,8 @@ public static String getSubmitUser(VarAccess stdVarAccess, String osUser, Set adminSet) { String enableSpecifyPRoxyUserStr = - stdVarAccess.getVar(String.class, AppKeys.LINKIS_CLIENT_NONCUSTOMIZABLE_ENABLE_PROXY_USER); + stdVarAccess.getVar(String.class, CliKeys.LINKIS_CLIENT_NONCUSTOMIZABLE_ENABLE_PROXY_USER); Boolean enableSpecifyProxyUser = Boolean.parseBoolean(enableSpecifyPRoxyUserStr) || adminSet.contains(submitUsr); @@ -129,8 +136,8 @@ public static String getProxyUser( String proxyUsr; if (enableSpecifyProxyUser) { - if (stdVarAccess.hasVar(AppKeys.JOB_COMMON_PROXY_USER)) { - proxyUsr = stdVarAccess.getVar(String.class, AppKeys.JOB_COMMON_PROXY_USER); + if (stdVarAccess.hasVar(CliKeys.JOB_COMMON_PROXY_USER)) { + proxyUsr = stdVarAccess.getVar(String.class, CliKeys.JOB_COMMON_PROXY_USER); if (!adminSet.contains(submitUsr) && adminSet.contains(proxyUsr)) { throw new BuilderException( "BLD0010", @@ -140,14 +147,14 @@ public static String getProxyUser( } } else { proxyUsr = submitUsr; - LogUtils.getInformationLogger() + LoggerManager.getInformationLogger() .info( "user does not specify proxy-user, will use current submit-user \"" + submitUsr + "\" by default."); } - } else if (stdVarAccess.hasVar(AppKeys.JOB_COMMON_PROXY_USER)) { - proxyUsr = stdVarAccess.getVar(String.class, AppKeys.JOB_COMMON_PROXY_USER); + } else if (stdVarAccess.hasVar(CliKeys.JOB_COMMON_PROXY_USER)) { + proxyUsr = stdVarAccess.getVar(String.class, CliKeys.JOB_COMMON_PROXY_USER); if (!StringUtils.equals(proxyUsr, submitUsr)) { throw new BuilderException( "BLD0010", @@ -157,7 +164,7 @@ public static String getProxyUser( } } else { proxyUsr = submitUsr; - LogUtils.getInformationLogger() + LoggerManager.getInformationLogger() .info( "user does not specify proxy-user, will use current submit-user \"" + proxyUsr @@ -199,4 +206,66 @@ public static String readFile(String path) { e); } } + + public static T castStringToAny(Class clazz, String val) { + if (StringUtils.isBlank(val)) { + return null; + } + T ret = null; + if (clazz == Object.class) { + ret = clazz.cast(val); + } else if (clazz == String.class) { + ret = clazz.cast(val); + } else if (clazz == Integer.class) { + ret = clazz.cast(Integer.parseInt(val)); + } else if (clazz == Double.class) { + ret = clazz.cast(Double.parseDouble(val)); + } else if (clazz == Float.class) { + ret = clazz.cast(Float.parseFloat(val)); + } else if (clazz == Long.class) { + ret = clazz.cast(Long.parseLong(val)); + } else if (clazz == Boolean.class) { + ret = clazz.cast(Boolean.parseBoolean(val)); + } + return ret; + } + + public static Map parseKVStringToMap(String kvStr, String separator) { + if (StringUtils.isBlank(separator)) { + separator = ","; + } + if (StringUtils.isBlank(kvStr)) { + return null; + } + Map argsProps = new HashMap<>(); + String[] args = StringUtils.splitByWholeSeparator(kvStr, separator); + for (String arg : args) { + int index = arg.indexOf("="); + if (index != -1) { + argsProps.put(arg.substring(0, index).trim(), arg.substring(index + 1).trim()); + } + } + + return argsProps; + } + + public static boolean isValidExecId(String execId) { + boolean ret = false; + if (StringUtils.isNotBlank(execId)) { + ret = true; + } + return ret; + } + + public static String progressInPercentage(float progress) { + return String.valueOf(progress * 100) + "%"; + } + + public static void doSleepQuietly(Long sleepMills) { + try { + Thread.sleep(sleepMills); + } catch (Exception ignore) { + // ignored + } + } } diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/utils/LogUtils.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/utils/LoggerManager.java similarity index 93% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/utils/LogUtils.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/utils/LoggerManager.java index b8147a4a60..bf302e9e85 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/utils/LogUtils.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/utils/LoggerManager.java @@ -15,12 +15,12 @@ * limitations under the License. */ -package org.apache.linkis.cli.core.utils; +package org.apache.linkis.cli.application.utils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class LogUtils { +public class LoggerManager { private static final Logger PlainTextLogger = LoggerFactory.getLogger("PlaintTextLogger"); private static final Logger InformationLogger = LoggerFactory.getLogger("InformationLogger"); diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/utils/SchedulerUtils.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/utils/SchedulerManager.java similarity index 94% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/utils/SchedulerUtils.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/utils/SchedulerManager.java index 5cfce48628..48aa367959 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/utils/SchedulerUtils.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/utils/SchedulerManager.java @@ -15,12 +15,12 @@ * limitations under the License. */ -package org.apache.linkis.cli.core.utils; +package org.apache.linkis.cli.application.utils; import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicInteger; -public class SchedulerUtils { +public class SchedulerManager { private static ExecutorService fixedThreadPool; private static ThreadPoolExecutor cachedThreadPool; private static int THREAD_NUM = 5; @@ -62,7 +62,7 @@ public static ExecutorService newFixedThreadPool( public static ThreadPoolExecutor getCachedThreadPoolExecutor() { if (cachedThreadPool == null) { - synchronized (SchedulerUtils.class) { + synchronized (SchedulerManager.class) { if (cachedThreadPool == null) { cachedThreadPool = newCachedThreadPool(THREAD_NUM, THREAD_NAME, IS_DEAMON); } @@ -73,7 +73,7 @@ public static ThreadPoolExecutor getCachedThreadPoolExecutor() { public static ExecutorService getFixedThreadPool() { if (fixedThreadPool == null) { - synchronized (SchedulerUtils.class) { + synchronized (SchedulerManager.class) { if (fixedThreadPool == null) { fixedThreadPool = newFixedThreadPool(THREAD_NUM, THREAD_NAME, IS_DEAMON); } diff --git a/linkis-computation-governance/linkis-client/linkis-cli/src/main/resources/version.properties b/linkis-computation-governance/linkis-client/linkis-cli/src/main/resources/version.properties new file mode 100644 index 0000000000..0da37e5c03 --- /dev/null +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/resources/version.properties @@ -0,0 +1,16 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +cli.version=${project.version} \ No newline at end of file diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/LinkisClientApplicationTest.java b/linkis-computation-governance/linkis-client/linkis-cli/src/test/java/org/apache/linkis/cli/application/LinkisClientApplicationTest.java similarity index 88% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/LinkisClientApplicationTest.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/test/java/org/apache/linkis/cli/application/LinkisClientApplicationTest.java index 0af2226661..d7ca2c0349 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/LinkisClientApplicationTest.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/test/java/org/apache/linkis/cli/application/LinkisClientApplicationTest.java @@ -28,26 +28,36 @@ public class LinkisClientApplicationTest { String[] cmdStr; String[] cmdStr2; + String[] cmdStr3; @BeforeEach public void before() { System.setProperty("conf.root", "src/test/resources/conf/"); System.setProperty("user.name", "hadoop"); + cmdStr3 = new String[] {"--version"}; cmdStr2 = new String[] { - // "--gatewayUrl", "http://127.0.0.1:8090", - // "--authStg", "token", - // "--authKey", "Validation-Code", - // "--authVal", "BML-AUTH", - // "job", - // "kill", - // "-j", "1121", - // "-submitUser", "user", - // "-proxyUser", "user", - - // "-varMap", "name=\"tables\"", - // "-varMap", "name=\"databases\"" - + "--gatewayUrl", + "http://127.0.0.1:9001", + "--authStg", + "token", + "--authKey", + "Validation-Code", + "--authVal", + "BML-AUTH", + "--status", + // "--log", + // "--kill", + // "--result", + "5773107", + "-submitUser", + "hadoop", + "-proxyUser", + "hadoop", + "-varMap", + "name=\"tables\"", + "-varMap", + "name2=\"databases\"" }; cmdStr = new String[] { @@ -99,6 +109,7 @@ public void before() { "-codeType", "shell", "-code", + // "exit -1", "whoami", // "-engineType", "spark-2.4.3", @@ -165,7 +176,8 @@ public void testProcessInput() { /** Method: exec(ProcessedData data) */ @Test public void testExec() { - // LinkisClientApplication.main(cmdStr); + // LinkisClientApplication.main(cmdStr); + LinkisClientApplication.main(cmdStr3); // LinkisClientApplication.main(cmdStr2); /* try { diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/constants/AppConstantsTest.java b/linkis-computation-governance/linkis-client/linkis-cli/src/test/java/org/apache/linkis/cli/application/constants/CliConstantsTest.java similarity index 71% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/constants/AppConstantsTest.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/test/java/org/apache/linkis/cli/application/constants/CliConstantsTest.java index bb16d43e47..b576c89f4f 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/constants/AppConstantsTest.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/test/java/org/apache/linkis/cli/application/constants/CliConstantsTest.java @@ -21,25 +21,25 @@ import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; -public class AppConstantsTest { +public class CliConstantsTest { @Test @DisplayName("constTest") public void constTest() { - String defaultConfigName = AppConstants.DEFAULT_CONFIG_NAME; - Long jobQuerySleepMills = AppConstants.JOB_QUERY_SLEEP_MILLS; - String resultsetLogo = AppConstants.RESULTSET_LOGO; - String resultsetMetaBeginLogo = AppConstants.RESULTSET_META_BEGIN_LOGO; - String resultsetMetaEndLogo = AppConstants.RESULTSET_META_END_LOGO; - String resultsetSeparatorLogo = AppConstants.RESULTSET_SEPARATOR_LOGO; - Integer resultsetPageSize = AppConstants.RESULTSET_PAGE_SIZE; - String jobCreatorDefault = AppConstants.JOB_CREATOR_DEFAULT; - String jobCreatorAsyncDefault = AppConstants.JOB_CREATOR_ASYNC_DEFAULT; - String dummyCid = AppConstants.DUMMY_CID; - String linkisCli = AppConstants.LINKIS_CLI; - String ujesMode = AppConstants.UJES_MODE; - String onceMode = AppConstants.ONCE_MODE; + String defaultConfigName = CliConstants.DEFAULT_CONFIG_NAME; + Long jobQuerySleepMills = CliConstants.JOB_QUERY_SLEEP_MILLS; + String resultsetLogo = CliConstants.RESULTSET_LOGO; + String resultsetMetaBeginLogo = CliConstants.RESULTSET_META_BEGIN_LOGO; + String resultsetMetaEndLogo = CliConstants.RESULTSET_META_END_LOGO; + String resultsetSeparatorLogo = CliConstants.RESULTSET_SEPARATOR_LOGO; + Integer resultsetPageSize = CliConstants.RESULTSET_PAGE_SIZE; + String jobCreatorDefault = CliConstants.JOB_CREATOR_DEFAULT; + String jobCreatorAsyncDefault = CliConstants.JOB_CREATOR_ASYNC_DEFAULT; + String dummyCid = CliConstants.DUMMY_CID; + String linkisCli = CliConstants.LINKIS_CLI; + String ujesMode = CliConstants.UJES_MODE; + String onceMode = CliConstants.ONCE_MODE; Assertions.assertEquals("linkis-cli.properties", defaultConfigName); Assertions.assertTrue(2000L == jobQuerySleepMills.longValue()); diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/constants/AppKeysTest.java b/linkis-computation-governance/linkis-client/linkis-cli/src/test/java/org/apache/linkis/cli/application/constants/CliKeysTest.java similarity index 73% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/constants/AppKeysTest.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/test/java/org/apache/linkis/cli/application/constants/CliKeysTest.java index f2477149b8..3fc7b84a0a 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/constants/AppKeysTest.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/test/java/org/apache/linkis/cli/application/constants/CliKeysTest.java @@ -21,27 +21,27 @@ import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; -public class AppKeysTest { +public class CliKeysTest { @Test @DisplayName("constTest") public void constTest() { - String adminUsers = AppKeys.ADMIN_USERS; - String linkisClientNoncustomizable = AppKeys.LINKIS_CLIENT_NONCUSTOMIZABLE; - String logPathKey = AppKeys.LOG_PATH_KEY; - String logFileKey = AppKeys.LOG_FILE_KEY; - String clientConfigRootKey = AppKeys.CLIENT_CONFIG_ROOT_KEY; - String defaultConfigFileNameKey = AppKeys.DEFAULT_CONFIG_FILE_NAME_KEY; - String linuxUserKey = AppKeys.LINUX_USER_KEY; - String jobExec = AppKeys.JOB_EXEC; - String jobExecCode = AppKeys.JOB_EXEC_CODE; - String jobContent = AppKeys.JOB_CONTENT; - String jobSource = AppKeys.JOB_SOURCE; - String jobParamConf = AppKeys.JOB_PARAM_CONF; - String jobParamRuntime = AppKeys.JOB_PARAM_RUNTIME; - String jobParamVar = AppKeys.JOB_PARAM_VAR; - String jobLabel = AppKeys.JOB_LABEL; + String adminUsers = CliKeys.ADMIN_USERS; + String linkisClientNoncustomizable = CliKeys.LINKIS_CLIENT_NONCUSTOMIZABLE; + String logPathKey = CliKeys.LOG_PATH_KEY; + String logFileKey = CliKeys.LOG_FILE_KEY; + String clientConfigRootKey = CliKeys.CLIENT_CONFIG_ROOT_KEY; + String defaultConfigFileNameKey = CliKeys.DEFAULT_CONFIG_FILE_NAME_KEY; + String linuxUserKey = CliKeys.LINUX_USER_KEY; + String jobExec = CliKeys.JOB_EXEC; + String jobExecCode = CliKeys.JOB_EXEC_CODE; + String jobContent = CliKeys.JOB_CONTENT; + String jobSource = CliKeys.JOB_SOURCE; + String jobParamConf = CliKeys.JOB_PARAM_CONF; + String jobParamRuntime = CliKeys.JOB_PARAM_RUNTIME; + String jobParamVar = CliKeys.JOB_PARAM_VAR; + String jobLabel = CliKeys.JOB_LABEL; Assertions.assertEquals("hadoop,root,shangda", adminUsers); Assertions.assertEquals("wds.linkis.client.noncustomizable", linkisClientNoncustomizable); diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/constants/LinkisConstantsTest.java b/linkis-computation-governance/linkis-client/linkis-cli/src/test/java/org/apache/linkis/cli/application/constants/LinkisConstantsTest.java similarity index 100% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/constants/LinkisConstantsTest.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/test/java/org/apache/linkis/cli/application/constants/LinkisConstantsTest.java diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/constants/LinkisKeysTest.java b/linkis-computation-governance/linkis-client/linkis-cli/src/test/java/org/apache/linkis/cli/application/constants/LinkisKeysTest.java similarity index 100% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/constants/LinkisKeysTest.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/test/java/org/apache/linkis/cli/application/constants/LinkisKeysTest.java diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/constants/TestConstants.java b/linkis-computation-governance/linkis-client/linkis-cli/src/test/java/org/apache/linkis/cli/application/constants/TestConstants.java similarity index 100% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/constants/TestConstants.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/test/java/org/apache/linkis/cli/application/constants/TestConstants.java diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/interactor/command/LinkisCmdTypeTest.java b/linkis-computation-governance/linkis-client/linkis-cli/src/test/java/org/apache/linkis/cli/application/interactor/command/CliCmdTypeTest.java similarity index 86% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/interactor/command/LinkisCmdTypeTest.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/test/java/org/apache/linkis/cli/application/interactor/command/CliCmdTypeTest.java index 16f47274d4..fecda11db7 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/interactor/command/LinkisCmdTypeTest.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/test/java/org/apache/linkis/cli/application/interactor/command/CliCmdTypeTest.java @@ -21,15 +21,15 @@ import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; -public class LinkisCmdTypeTest { +public class CliCmdTypeTest { @Test @DisplayName("enumTest") public void enumTest() { - int id = LinkisCmdType.UNIVERSAL.getId(); - String name = LinkisCmdType.UNIVERSAL.getName(); - String desc = LinkisCmdType.UNIVERSAL.getDesc(); + int id = CliCmdType.UNIVERSAL.getId(); + String name = CliCmdType.UNIVERSAL.getName(); + String desc = CliCmdType.UNIVERSAL.getDesc(); Assertions.assertTrue(1 == id); Assertions.assertEquals("linkis-cli", name); diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/interactor/command/template/TestCmdType.java b/linkis-computation-governance/linkis-client/linkis-cli/src/test/java/org/apache/linkis/cli/application/interactor/command/template/TestCmdType.java similarity index 96% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/interactor/command/template/TestCmdType.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/test/java/org/apache/linkis/cli/application/interactor/command/template/TestCmdType.java index 54a5952f09..0155ee5029 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/interactor/command/template/TestCmdType.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/test/java/org/apache/linkis/cli/application/interactor/command/template/TestCmdType.java @@ -18,7 +18,7 @@ package org.apache.linkis.cli.application.interactor.command.template; import org.apache.linkis.cli.application.constants.TestConstants; -import org.apache.linkis.cli.common.entity.command.CmdType; +import org.apache.linkis.cli.application.entity.command.CmdType; public enum TestCmdType implements CmdType { TEST_PRIMARY(TestConstants.PRIMARY_COMMAND, 1, TestConstants.SPARK_DESC), diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/interactor/command/template/TestParamMapper.java b/linkis-computation-governance/linkis-client/linkis-cli/src/test/java/org/apache/linkis/cli/application/interactor/command/template/TestParamMapper.java similarity index 78% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/interactor/command/template/TestParamMapper.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/test/java/org/apache/linkis/cli/application/interactor/command/template/TestParamMapper.java index 1a8959f097..a077cfcca7 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/interactor/command/template/TestParamMapper.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/test/java/org/apache/linkis/cli/application/interactor/command/template/TestParamMapper.java @@ -17,19 +17,19 @@ package org.apache.linkis.cli.application.interactor.command.template; -import org.apache.linkis.cli.application.constants.AppKeys; +import org.apache.linkis.cli.application.constants.CliKeys; import org.apache.linkis.cli.application.constants.TestConstants; -import org.apache.linkis.cli.core.interactor.command.parser.transformer.ParamKeyMapper; +import org.apache.linkis.cli.application.interactor.command.parser.transformer.ParamKeyMapper; public class TestParamMapper extends ParamKeyMapper { @Override public void initMapperRules() { - super.updateMapping(TestConstants.PARAM_COMMON_CMD, AppKeys.JOB_EXEC_CODE); - super.updateMapping(TestConstants.PARAM_PROXY, AppKeys.LINKIS_COMMON_GATEWAY_URL); - super.updateMapping(TestConstants.PARAM_USER, AppKeys.LINKIS_COMMON_TOKEN_KEY); - super.updateMapping(TestConstants.PARAM_USR_CONF, AppKeys.LINKIS_CLIENT_USER_CONFIG); - super.updateMapping(TestConstants.PARAM_PASSWORD, AppKeys.LINKIS_COMMON_TOKEN_VALUE); - super.updateMapping(TestConstants.PARAM_PROXY_USER, AppKeys.JOB_COMMON_PROXY_USER); + super.updateMapping(TestConstants.PARAM_COMMON_CMD, CliKeys.JOB_EXEC_CODE); + super.updateMapping(TestConstants.PARAM_PROXY, CliKeys.LINKIS_COMMON_GATEWAY_URL); + super.updateMapping(TestConstants.PARAM_USER, CliKeys.LINKIS_COMMON_TOKEN_KEY); + super.updateMapping(TestConstants.PARAM_USR_CONF, CliKeys.LINKIS_CLIENT_USER_CONFIG); + super.updateMapping(TestConstants.PARAM_PASSWORD, CliKeys.LINKIS_COMMON_TOKEN_VALUE); + super.updateMapping(TestConstants.PARAM_PROXY_USER, CliKeys.JOB_COMMON_PROXY_USER); updateMapping( TestConstants.PARAM_SPARK_EXECUTOR_CORES, TestConstants.LINKIS_SPARK_EXECUTOR_CORES); @@ -43,7 +43,7 @@ public void initMapperRules() { updateMapping( TestConstants.PARAM_SPARK_SHUFFLE_PARTITIONS, TestConstants.LINKIS_SPARK_SHUFFLE_PARTITIONS); - updateMapping(TestConstants.PARAM_SPARK_RUNTYPE, AppKeys.JOB_LABEL_CODE_TYPE); + updateMapping(TestConstants.PARAM_SPARK_RUNTYPE, CliKeys.JOB_LABEL_CODE_TYPE); updateMapping(TestConstants.PARAM_YARN_QUEUE, TestConstants.YARN_QUEUE); } diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/interactor/command/template/TestSparkCmdTemplate.java b/linkis-computation-governance/linkis-client/linkis-cli/src/test/java/org/apache/linkis/cli/application/interactor/command/template/TestSparkCmdTemplate.java similarity index 94% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/interactor/command/template/TestSparkCmdTemplate.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/test/java/org/apache/linkis/cli/application/interactor/command/template/TestSparkCmdTemplate.java index c9274fbf71..fb53803b6a 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/interactor/command/template/TestSparkCmdTemplate.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/test/java/org/apache/linkis/cli/application/interactor/command/template/TestSparkCmdTemplate.java @@ -17,11 +17,10 @@ package org.apache.linkis.cli.application.interactor.command.template; -import org.apache.linkis.cli.application.constants.AppKeys; +import org.apache.linkis.cli.application.constants.CliKeys; import org.apache.linkis.cli.application.constants.TestConstants; -import org.apache.linkis.cli.core.exception.CommandException; -import org.apache.linkis.cli.core.interactor.command.template.AbstractCmdTemplate; -import org.apache.linkis.cli.core.interactor.command.template.option.StdOption; +import org.apache.linkis.cli.application.exception.CommandException; +import org.apache.linkis.cli.application.interactor.command.template.option.StdOption; import java.util.HashMap; import java.util.Map; @@ -50,7 +49,7 @@ public class TestSparkCmdTemplate extends AbstractCmdTemplate { protected StdOption userOp = option( null, - AppKeys.JOB_COMMON_SUBMIT_USER, + CliKeys.JOB_COMMON_SUBMIT_USER, new String[] {"-u", "--user"}, "specify user", true, @@ -67,7 +66,7 @@ public class TestSparkCmdTemplate extends AbstractCmdTemplate { private StdOption passwordOp = option( null, - AppKeys.JOB_COMMON_SUBMIT_PASSWORD, + CliKeys.JOB_COMMON_SUBMIT_PASSWORD, new String[] {"-pwd", "--passwd"}, "specify user password", true, @@ -95,7 +94,7 @@ public class TestSparkCmdTemplate extends AbstractCmdTemplate { private StdOption> confMap = option( null, - AppKeys.JOB_PARAM_CONF, + CliKeys.JOB_PARAM_CONF, new String[] {"-confMap"}, "confMap", true, diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/interactor/job/LinkisJobStatusTest.java b/linkis-computation-governance/linkis-client/linkis-cli/src/test/java/org/apache/linkis/cli/application/interactor/job/LinkisJobStatusTest.java similarity index 95% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/interactor/job/LinkisJobStatusTest.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/test/java/org/apache/linkis/cli/application/interactor/job/LinkisJobStatusTest.java index 1b3b7c67ec..ff71dd0c43 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/interactor/job/LinkisJobStatusTest.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/test/java/org/apache/linkis/cli/application/interactor/job/LinkisJobStatusTest.java @@ -17,6 +17,8 @@ package org.apache.linkis.cli.application.interactor.job; +import org.apache.linkis.cli.application.interactor.job.common.LinkisJobStatus; + import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/interactor/job/builder/ProcessKeyUtilsTest.java b/linkis-computation-governance/linkis-client/linkis-cli/src/test/java/org/apache/linkis/cli/application/interactor/job/builder/KeyParserTest.java similarity index 81% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/interactor/job/builder/ProcessKeyUtilsTest.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/test/java/org/apache/linkis/cli/application/interactor/job/builder/KeyParserTest.java index cc579bec17..4c6c494c20 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/interactor/job/builder/ProcessKeyUtilsTest.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/test/java/org/apache/linkis/cli/application/interactor/job/builder/KeyParserTest.java @@ -17,7 +17,8 @@ package org.apache.linkis.cli.application.interactor.job.builder; -import org.apache.linkis.cli.application.constants.AppKeys; +import org.apache.linkis.cli.application.constants.CliKeys; +import org.apache.linkis.cli.application.interactor.job.common.KeyParser; import java.util.HashMap; import java.util.Map; @@ -26,17 +27,17 @@ import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; -public class ProcessKeyUtilsTest { +public class KeyParserTest { @Test @DisplayName("removePrefixForKeysInMapTest") public void removePrefixForKeysInMapTest() { Map map = new HashMap<>(); - map.put(AppKeys.JOB_PARAM_CONF, new Object()); + map.put(CliKeys.JOB_PARAM_CONF, new Object()); map.put("name", new Object()); - Map stringObjectMap = ProcessKeyUtils.removePrefixForKeysInMap(map); + Map stringObjectMap = KeyParser.removePrefixForKeysInMap(map); Assertions.assertTrue(1 == stringObjectMap.size()); } } diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/interactor/job/subtype/LinkisManSubTypeTest.java b/linkis-computation-governance/linkis-client/linkis-cli/src/test/java/org/apache/linkis/cli/application/interactor/job/subtype/JobCmdSubTypeTest.java similarity index 75% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/interactor/job/subtype/LinkisManSubTypeTest.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/test/java/org/apache/linkis/cli/application/interactor/job/subtype/JobCmdSubTypeTest.java index 699eaaf8b0..c720e7f2c2 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/interactor/job/subtype/LinkisManSubTypeTest.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/test/java/org/apache/linkis/cli/application/interactor/job/subtype/JobCmdSubTypeTest.java @@ -17,22 +17,24 @@ package org.apache.linkis.cli.application.interactor.job.subtype; +import org.apache.linkis.cli.application.interactor.job.jobcmd.JobCmdSubType; + import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; -public class LinkisManSubTypeTest { +public class JobCmdSubTypeTest { @Test @DisplayName("enumTest") public void enumTest() { - String killName = LinkisManSubType.KILL.getName(); - String logName = LinkisManSubType.LOG.getName(); - String descName = LinkisManSubType.DESC.getName(); - String statusName = LinkisManSubType.STATUS.getName(); - String listName = LinkisManSubType.LIST.getName(); - String resultName = LinkisManSubType.RESULT.getName(); + String killName = JobCmdSubType.KILL.getName(); + String logName = JobCmdSubType.LOG.getName(); + String descName = JobCmdSubType.DESC.getName(); + String statusName = JobCmdSubType.STATUS.getName(); + String listName = JobCmdSubType.LIST.getName(); + String resultName = JobCmdSubType.RESULT.getName(); Assertions.assertEquals("kill", killName); Assertions.assertEquals("log", logName); diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/operator/once/OnceJobConstantsTest.java b/linkis-computation-governance/linkis-client/linkis-cli/src/test/java/org/apache/linkis/cli/application/operator/once/OnceJobConstantsTest.java similarity index 100% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/operator/once/OnceJobConstantsTest.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/test/java/org/apache/linkis/cli/application/operator/once/OnceJobConstantsTest.java diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/operator/ujes/UJESConstantsTest.java b/linkis-computation-governance/linkis-client/linkis-cli/src/test/java/org/apache/linkis/cli/application/operator/ujes/UJESConstantsTest.java similarity index 100% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/operator/ujes/UJESConstantsTest.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/test/java/org/apache/linkis/cli/application/operator/ujes/UJESConstantsTest.java diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/utils/UtilsTest.java b/linkis-computation-governance/linkis-client/linkis-cli/src/test/java/org/apache/linkis/cli/application/utils/UtilsTest.java similarity index 95% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/utils/UtilsTest.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/test/java/org/apache/linkis/cli/application/utils/UtilsTest.java index 5af0e6ab85..8e07a5a6bb 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/utils/UtilsTest.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/test/java/org/apache/linkis/cli/application/utils/UtilsTest.java @@ -28,7 +28,7 @@ public class UtilsTest { public void isValidExecIdTest() { String execId = "0001"; - boolean validExecId = Utils.isValidExecId(execId); + boolean validExecId = CliUtils.isValidExecId(execId); Assertions.assertTrue(validExecId); } } diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/resources/conf/linkis-cli.properties b/linkis-computation-governance/linkis-client/linkis-cli/src/test/resources/conf/linkis-cli.properties similarity index 100% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/resources/conf/linkis-cli.properties rename to linkis-computation-governance/linkis-client/linkis-cli/src/test/resources/conf/linkis-cli.properties diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/resources/conf/user.properties b/linkis-computation-governance/linkis-client/linkis-cli/src/test/resources/conf/user.properties similarity index 100% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/resources/conf/user.properties rename to linkis-computation-governance/linkis-client/linkis-cli/src/test/resources/conf/user.properties diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/resources/linkis-cli.properties b/linkis-computation-governance/linkis-client/linkis-cli/src/test/resources/linkis-cli.properties similarity index 100% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/resources/linkis-cli.properties rename to linkis-computation-governance/linkis-client/linkis-cli/src/test/resources/linkis-cli.properties diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/resources/log4j2.xml b/linkis-computation-governance/linkis-client/linkis-cli/src/test/resources/log4j2.xml similarity index 100% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/resources/log4j2.xml rename to linkis-computation-governance/linkis-client/linkis-cli/src/test/resources/log4j2.xml diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/resources/testScala.scala b/linkis-computation-governance/linkis-client/linkis-cli/src/test/resources/testScala.scala similarity index 100% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/resources/testScala.scala rename to linkis-computation-governance/linkis-client/linkis-cli/src/test/resources/testScala.scala diff --git a/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/once/LinkisManagerClient.scala b/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/once/LinkisManagerClient.scala index 3ac3cb7c88..45f3f49bed 100644 --- a/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/once/LinkisManagerClient.scala +++ b/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/once/LinkisManagerClient.scala @@ -17,7 +17,9 @@ package org.apache.linkis.computation.client.once +import org.apache.linkis.common.utils.Utils import org.apache.linkis.computation.client.once.action.{ + AskEngineConnAction, CreateEngineConnAction, EngineConnOperateAction, GetEngineConnAction, @@ -25,6 +27,7 @@ import org.apache.linkis.computation.client.once.action.{ LinkisManagerAction } import org.apache.linkis.computation.client.once.result.{ + AskEngineConnResult, CreateEngineConnResult, EngineConnOperateResult, GetEngineConnResult, @@ -39,6 +42,8 @@ import java.io.Closeable trait LinkisManagerClient extends Closeable { + def askEngineConn(askEngineConnAction: AskEngineConnAction): AskEngineConnResult + def createEngineConn(createEngineConnAction: CreateEngineConnAction): CreateEngineConnResult def getEngineConn(getEngineConnAction: GetEngineConnAction): GetEngineConnResult @@ -82,7 +87,21 @@ class LinkisManagerClientImpl(ujesClient: UJESClient) extends LinkisManagerClien override def executeEngineConnOperation( engineConnOperateAction: EngineConnOperateAction - ): EngineConnOperateResult = execute(engineConnOperateAction) + ): EngineConnOperateResult = { + Utils.tryCatch { + val rs = execute[EngineConnOperateResult](engineConnOperateAction) + rs + } { case e: Exception => + val rs = new EngineConnOperateResult + rs.setIsError(true) + rs.setErrorMsg(e.getMessage) + rs + } + } override def close(): Unit = ujesClient.close() + + override def askEngineConn(askEngineConnAction: AskEngineConnAction): AskEngineConnResult = + execute(askEngineConnAction) + } diff --git a/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/once/action/AskEngineConnAction.scala b/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/once/action/AskEngineConnAction.scala new file mode 100644 index 0000000000..4b89b53764 --- /dev/null +++ b/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/once/action/AskEngineConnAction.scala @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.computation.client.once.action + +import org.apache.linkis.httpclient.dws.DWSHttpClient +import org.apache.linkis.httpclient.request.POSTAction +import org.apache.linkis.ujes.client.exception.UJESJobException + +import org.apache.commons.lang3.StringUtils + +import java.util + +class AskEngineConnAction extends POSTAction with LinkisManagerAction { + + override def getRequestPayload: String = + DWSHttpClient.jacksonJson.writeValueAsString(getRequestPayloads) + + override def suffixURLs: Array[String] = Array("linkisManager", "askEngineConn") + +} + +object AskEngineConnAction { + + def newBuilder(): Builder = new Builder + + class Builder private[AskEngineConnAction] () { + private var user: String = _ + private var properties: util.Map[String, String] = _ + private var labels: util.Map[String, String] = _ + private var maxSubmitTime: Long = _ + private var createService: String = _ + private var description: String = _ + + def setUser(user: String): Builder = { + this.user = user + this + } + + def setProperties(properties: util.Map[String, String]): Builder = { + this.properties = properties + this + } + + def setLabels(labels: java.util.Map[String, String]): Builder = { + this.labels = labels + this + } + + def setMaxSubmitTime(maxSubmitTime: Long): Builder = { + this.maxSubmitTime = maxSubmitTime + this + } + + def setCreateService(createService: String): Builder = { + this.createService = createService + this + } + + def setDescription(description: String): Builder = { + this.description = description + this + } + + def build(): AskEngineConnAction = { + val action = new AskEngineConnAction() + if (user == null) throw new UJESJobException("user is needed!") + if (properties == null) properties = new java.util.HashMap[String, String] + if (labels == null) throw new UJESJobException("labels is needed!") + action.setUser(user) + action.addRequestPayload("properties", properties) + action.addRequestPayload("labels", labels) + if (StringUtils.isNotBlank(createService)) { + action.addRequestPayload("createService", createService) + } + if (null != maxSubmitTime) { + action.addRequestPayload("timeOut", maxSubmitTime) + } + if (StringUtils.isNotBlank(description)) { + action.addRequestPayload("description", description) + } + action + } + + } + +} diff --git a/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/once/result/AskEngineConnResult.scala b/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/once/result/AskEngineConnResult.scala new file mode 100644 index 0000000000..58c6085b45 --- /dev/null +++ b/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/once/result/AskEngineConnResult.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.computation.client.once.result + +import org.apache.linkis.httpclient.dws.annotation.DWSHttpMessageResult + +@DWSHttpMessageResult("/api/rest_j/v\\d+/linkisManager/askEngineConn") +class AskEngineConnResult extends GetEngineConnResult diff --git a/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/once/result/EngineConnOperateResult.scala b/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/once/result/EngineConnOperateResult.scala index 1bf12e0418..50df73bd10 100644 --- a/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/once/result/EngineConnOperateResult.scala +++ b/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/once/result/EngineConnOperateResult.scala @@ -33,9 +33,11 @@ class EngineConnOperateResult extends LinkisManagerResult { this.result = result } + def getErrorMsg(): String = errorMsg + def setErrorMsg(errorMsg: String): Unit = this.errorMsg = errorMsg - def setError(isError: Boolean): Unit = this.isError = isError + def getIsError(): Boolean = isError def setIsError(isError: Boolean): Unit = this.isError = isError diff --git a/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/once/simple/SimpleOnceJob.scala b/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/once/simple/SimpleOnceJob.scala index baab361b58..e672e7bfe1 100644 --- a/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/once/simple/SimpleOnceJob.scala +++ b/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/once/simple/SimpleOnceJob.scala @@ -94,6 +94,10 @@ trait SimpleOnceJob extends OnceJob { case operator => operator } + def getEcServiceInstance: ServiceInstance = serviceInstance + + def getEcTicketId: String = ticketId + } class SubmittableSimpleOnceJob( diff --git a/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/operator/impl/EngineConnApplicationInfoOperator.scala b/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/operator/impl/EngineConnApplicationInfoOperator.scala index 83399bf371..a1dba63404 100644 --- a/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/operator/impl/EngineConnApplicationInfoOperator.scala +++ b/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/operator/impl/EngineConnApplicationInfoOperator.scala @@ -19,6 +19,7 @@ package org.apache.linkis.computation.client.operator.impl import org.apache.linkis.computation.client.once.result.EngineConnOperateResult import org.apache.linkis.computation.client.operator.OnceJobOperator +import org.apache.linkis.governance.common.constant.ec.ECConstants import org.apache.linkis.ujes.client.exception.UJESJobException class EngineConnApplicationInfoOperator extends OnceJobOperator[ApplicationInfo] { @@ -28,7 +29,7 @@ class EngineConnApplicationInfoOperator extends OnceJobOperator[ApplicationInfo] override protected def resultToObject(result: EngineConnOperateResult): ApplicationInfo = { ApplicationInfo( result - .getAsOption("applicationId") + .getAsOption(ECConstants.YARN_APPID_NAME_KEY) .getOrElse( throw new UJESJobException( 20300, @@ -36,14 +37,14 @@ class EngineConnApplicationInfoOperator extends OnceJobOperator[ApplicationInfo] ) ), result - .getAsOption("applicationUrl") + .getAsOption(ECConstants.YARN_APP_URL_KEY) .getOrElse( throw new UJESJobException( 20300, s"Cannot get applicationUrl from EngineConn $getServiceInstance." ) ), - result.getAs("queue") + result.getAs(ECConstants.QUEUE) ) } diff --git a/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/ujes/client/request/ResultSetAction.scala b/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/ujes/client/request/ResultSetAction.scala index 6b41b4c62b..9eb748691e 100644 --- a/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/ujes/client/request/ResultSetAction.scala +++ b/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/ujes/client/request/ResultSetAction.scala @@ -35,6 +35,9 @@ object ResultSetAction { private var pageSize: Int = _ private var charset: String = Configuration.BDP_ENCODING.getValue + // default value is :org.apache.linkis.storage.domain.Dolphin.LINKIS_NULL + private var nullValue: String = "LINKIS_NULL" + def setUser(user: String): Builder = { this.user = user this @@ -60,6 +63,11 @@ object ResultSetAction { this } + def setNullValue(nullValue: String): Builder = { + this.nullValue = nullValue + this + } + def build(): ResultSetAction = { if (user == null) throw new UJESClientBuilderException("user is needed!") if (path == null) throw new UJESClientBuilderException("path is needed!") @@ -68,6 +76,7 @@ object ResultSetAction { if (page > 0) resultSetAction.setParameter("page", page) if (pageSize > 0) resultSetAction.setParameter("pageSize", pageSize) resultSetAction.setParameter("charset", charset) + resultSetAction.setParameter("nullValue", nullValue) resultSetAction.setUser(user) resultSetAction } diff --git a/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/ujes/client/response/OpenLogResult.scala b/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/ujes/client/response/OpenLogResult.scala index fe107a32ca..2de5758aea 100644 --- a/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/ujes/client/response/OpenLogResult.scala +++ b/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/ujes/client/response/OpenLogResult.scala @@ -26,7 +26,7 @@ import scala.beans.BeanProperty class OpenLogResult extends DWSResult { /** - * log[0] - info log[1] - warn log[2] - error log[3] - all (info + warn + error) + * log[0] - error log[1] - warn log[2] - info log[3] - all (info + warn + error) */ @BeanProperty var log: Array[String] = _ diff --git a/linkis-computation-governance/linkis-computation-governance-common/src/main/java/org/apache/linkis/governance/common/entity/TemplateConfKey.java b/linkis-computation-governance/linkis-computation-governance-common/src/main/java/org/apache/linkis/governance/common/entity/TemplateConfKey.java new file mode 100644 index 0000000000..13cbac5577 --- /dev/null +++ b/linkis-computation-governance/linkis-computation-governance-common/src/main/java/org/apache/linkis/governance/common/entity/TemplateConfKey.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.governance.common.entity; + +public class TemplateConfKey { + + private String templateUuid; + + private String key; + + private String templateName; + + private String configValue; + + public String getTemplateUuid() { + return templateUuid; + } + + public void setTemplateUuid(String templateUuid) { + this.templateUuid = templateUuid; + } + + public String getKey() { + return key; + } + + public void setKey(String key) { + this.key = key; + } + + public String getTemplateName() { + return templateName; + } + + public void setTemplateName(String templateName) { + this.templateName = templateName; + } + + public String getConfigValue() { + return configValue; + } + + public void setConfigValue(String configValue) { + this.configValue = configValue; + } + + @Override + public String toString() { + return "TemplateKey{" + + "templateUuid='" + + templateUuid + + '\'' + + ", key='" + + key + + '\'' + + ", templateName='" + + templateName + + '\'' + + ", configValue='" + + configValue + + '\'' + + '}'; + } +} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/subtype/LinkisSubmitSubType.java b/linkis-computation-governance/linkis-computation-governance-common/src/main/java/org/apache/linkis/governance/common/enums/OnceJobOperationBoundary.java similarity index 78% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/subtype/LinkisSubmitSubType.java rename to linkis-computation-governance/linkis-computation-governance-common/src/main/java/org/apache/linkis/governance/common/enums/OnceJobOperationBoundary.java index 49d3d5ea91..37c6fc8d92 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/subtype/LinkisSubmitSubType.java +++ b/linkis-computation-governance/linkis-computation-governance-common/src/main/java/org/apache/linkis/governance/common/enums/OnceJobOperationBoundary.java @@ -15,20 +15,18 @@ * limitations under the License. */ -package org.apache.linkis.cli.application.interactor.job.subtype; +package org.apache.linkis.governance.common.enums; -import org.apache.linkis.cli.common.entity.job.JobSubType; - -public enum LinkisSubmitSubType implements JobSubType { - SUBMIT("submit"); +public enum OnceJobOperationBoundary { + ECM("ecm"), + EC("ec"); private String name; - LinkisSubmitSubType(String name) { + OnceJobOperationBoundary(String name) { this.name = name; } - @Override public String getName() { return name; } diff --git a/linkis-computation-governance/linkis-computation-governance-common/src/main/java/org/apache/linkis/governance/common/exception/engineconn/EngineConnExecutorErrorCode.java b/linkis-computation-governance/linkis-computation-governance-common/src/main/java/org/apache/linkis/governance/common/exception/engineconn/EngineConnExecutorErrorCode.java index c0d295755a..89d3c9eba4 100644 --- a/linkis-computation-governance/linkis-computation-governance-common/src/main/java/org/apache/linkis/governance/common/exception/engineconn/EngineConnExecutorErrorCode.java +++ b/linkis-computation-governance/linkis-computation-governance-common/src/main/java/org/apache/linkis/governance/common/exception/engineconn/EngineConnExecutorErrorCode.java @@ -37,4 +37,6 @@ public class EngineConnExecutorErrorCode { public static final int SEND_TO_ENTRANCE_ERROR = 40105; public static final int INIT_EXECUTOR_FAILED = 40106; + + public static final int INVALID_APPLICATION_ID = 40107; } diff --git a/linkis-computation-governance/linkis-computation-governance-common/src/main/java/org/apache/linkis/governance/common/protocol/conf/TemplateConfRequest.java b/linkis-computation-governance/linkis-computation-governance-common/src/main/java/org/apache/linkis/governance/common/protocol/conf/TemplateConfRequest.java new file mode 100644 index 0000000000..258f724e82 --- /dev/null +++ b/linkis-computation-governance/linkis-computation-governance-common/src/main/java/org/apache/linkis/governance/common/protocol/conf/TemplateConfRequest.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.governance.common.protocol.conf; + +import org.apache.linkis.protocol.message.RequestProtocol; + +public class TemplateConfRequest implements RequestProtocol { + + private String templateUuid; + + public TemplateConfRequest(String templateUuid) { + this.templateUuid = templateUuid; + } + + public String getTemplateUuid() { + return templateUuid; + } + + public void setTemplateUuid(String templateUuid) { + this.templateUuid = templateUuid; + } +} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/data/LinkisResultData.java b/linkis-computation-governance/linkis-computation-governance-common/src/main/java/org/apache/linkis/governance/common/protocol/conf/TemplateConfResponse.java similarity index 67% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/data/LinkisResultData.java rename to linkis-computation-governance/linkis-computation-governance-common/src/main/java/org/apache/linkis/governance/common/protocol/conf/TemplateConfResponse.java index f8e61a790d..8822fe988d 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/data/LinkisResultData.java +++ b/linkis-computation-governance/linkis-computation-governance-common/src/main/java/org/apache/linkis/governance/common/protocol/conf/TemplateConfResponse.java @@ -15,25 +15,22 @@ * limitations under the License. */ -package org.apache.linkis.cli.application.interactor.job.data; +package org.apache.linkis.governance.common.protocol.conf; -import java.util.List; - -public interface LinkisResultData extends LinkisJobData, Cloneable { - - String getResultLocation(); +import org.apache.linkis.governance.common.entity.TemplateConfKey; - String[] getResultSetPaths(); - - Boolean hasNextResultPage(); - - List consumeResultContent(); +import java.util.ArrayList; +import java.util.List; - void sendResultFin(); +public class TemplateConfResponse { - boolean resultFinReceived(); + private List list = new ArrayList<>(); - boolean hasResult(); + public List getList() { + return list; + } - void setHasResult(boolean hasResult); + public void setList(List list) { + this.list = list; + } } diff --git a/linkis-computation-governance/linkis-computation-governance-common/src/main/java/org/apache/linkis/governance/common/utils/LoggerUtils.java b/linkis-computation-governance/linkis-computation-governance-common/src/main/java/org/apache/linkis/governance/common/utils/LoggerUtils.java new file mode 100644 index 0000000000..99addd4fed --- /dev/null +++ b/linkis-computation-governance/linkis-computation-governance-common/src/main/java/org/apache/linkis/governance/common/utils/LoggerUtils.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.governance.common.utils; + +import org.apache.linkis.governance.common.conf.GovernanceCommonConf; +import org.apache.linkis.governance.common.constant.job.JobRequestConstants; + +import java.util.Map; + +import org.slf4j.MDC; + +public class LoggerUtils { + + public static void setJobIdMDC(String jobId) { + MDC.put(JobRequestConstants.JOB_ID(), jobId); + } + + public static void setJobIdMDC(Map props) { + if (GovernanceCommonConf.MDC_ENABLED()) { + String jobId = JobUtils.getJobIdFromMap(props); + MDC.put(JobRequestConstants.JOB_ID(), jobId); + } + } + + public static void removeJobIdMDC() { + MDC.remove(JobRequestConstants.JOB_ID()); + } +} diff --git a/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/conf/GovernaceCommonConf.scala b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/conf/GovernaceCommonConf.scala index 48f906bb8c..0d9b62ff48 100644 --- a/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/conf/GovernaceCommonConf.scala +++ b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/conf/GovernaceCommonConf.scala @@ -18,6 +18,7 @@ package org.apache.linkis.governance.common.conf import org.apache.linkis.common.conf.{CommonVars, Configuration} +import org.apache.linkis.governance.common.constant.ec.ECConstants object GovernanceCommonConf { @@ -40,10 +41,16 @@ object GovernanceCommonConf { val ENGINE_CONN_MANAGER_SPRING_NAME = CommonVars("wds.linkis.engineconn.manager.name", "linkis-cg-engineconnmanager") + val ENGINE_APPLICATION_MANAGER_SPRING_NAME = + CommonVars("wds.linkis.application.manager.name", "linkis-cg-linkismanager") + val ENGINE_CONN_PORT_RANGE = CommonVars("wds.linkis.engineconn.port.range", "-") val MANAGER_SERVICE_NAME = - CommonVars("wds.linkis.engineconn.manager.name", "linkis-cg-linkismanager") + CommonVars( + "wds.linkis.engineconn.manager.name", + GovernanceCommonConf.ENGINE_APPLICATION_MANAGER_SPRING_NAME.getValue + ) val ENTRANCE_SERVICE_NAME = CommonVars("wds.linkis.entrance.name", "linkis-cg-entrance") @@ -66,8 +73,17 @@ object GovernanceCommonConf { val ERROR_CODE_DESC_LEN = CommonVars("linkis.error.code.desc.len", 512, "Error code description maximum length").getValue + val FAKE_PROGRESS: Float = CommonVars[Float]("linkis.job.fake.progress", 0.99f).getValue + + val MDC_ENABLED = + CommonVars("linkis.mdc.log.enabled", true, "MDC Switch").getValue + def getEngineEnvValue(envKey: String): String = { CommonVars(envKey, "").getValue } + // value ECConstants.EC_CLIENT_TYPE_ATTACH + val EC_APP_MANAGE_MODE = + CommonVars("linkis.ec.app.manage.mode", "attach") + } diff --git a/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/constant/ec/ECConstants.scala b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/constant/ec/ECConstants.scala index fe48f6887d..a94eadf422 100644 --- a/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/constant/ec/ECConstants.scala +++ b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/constant/ec/ECConstants.scala @@ -35,4 +35,43 @@ object ECConstants { val YARN_QUEUE_NAME_CONFIG_KEY = "wds.linkis.rm.yarnqueue" + val QUEUE = "queue" + + val EC_CLIENT_TYPE_ATTACH = "attach" + + val EC_CLIENT_TYPE_DETACH = "detach" + + val YARN_APPID_NAME_KEY = "applicationId" + + val YARN_APP_URL_KEY = "applicationUrl" + + val YARN_APP_NAME_KEY = "appicationName" + + val YARN_MODE_KEY = "yarnMode" + + val EC_SERVICE_INSTANCE_KEY = "serviceInstance" + + val ECM_SERVICE_INSTANCE_KEY = "ecmServiceInstance" + + val MANAGER_SERVICE_INSTANCE_KEY = "managerServiceInstance" + + val NODE_STATUS_KEY = "nodeStatus" + + val EC_LAST_UNLOCK_TIMESTAMP = "lastUnlockTimestamp" + + val YARN_APP_TYPE_LIST_KEY = "yarnAppTypeList" + + val YARN_APP_STATE_LIST_KEY = "yarnAppStateList" + + val YARN_APP_TYPE_KEY = "yarnAppType" + + val YARN_APP_TYPE_SPARK = "spark" + + val YARN_APP_TYPE_FLINK = "flink" + + val EC_OPERATE_LIST = "list" + + val EC_OPERATE_STATUS = "status" + + val YARN_APP_RESULT_LIST_KEY = "yarnAppResultList" } diff --git a/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/paser/CodeParser.scala b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/paser/CodeParser.scala index 19bd7f9cdb..64ece62fd7 100644 --- a/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/paser/CodeParser.scala +++ b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/paser/CodeParser.scala @@ -86,6 +86,11 @@ abstract class CombinedEngineCodeParser extends CodeParser { } +/** + * Scala is no longer using Parser but instead using EmptyParser. If there is a comment at the end, + * it will cause the task to become stuck + */ +@deprecated class ScalaCodeParser extends SingleCodeParser with Logging { override val codeType: CodeType = CodeType.Scala @@ -109,6 +114,9 @@ class ScalaCodeParser extends SingleCodeParser with Logging { case _ => } if (statementBuffer.nonEmpty) codeBuffer.append(statementBuffer.mkString("\n")) + // Make sure the last line is not a comment + codeBuffer.append("\n") + codeBuffer.append("val linkisVar=123") codeBuffer.toArray } diff --git a/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/utils/ECPathUtils.scala b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/utils/ECPathUtils.scala index 58c08b1f84..236046f3d4 100644 --- a/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/utils/ECPathUtils.scala +++ b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/utils/ECPathUtils.scala @@ -17,6 +17,8 @@ package org.apache.linkis.governance.common.utils +import org.apache.linkis.manager.label.entity.engine.{EngineTypeLabel, UserCreatorLabel} + import org.apache.commons.lang3.StringUtils import org.apache.commons.lang3.time.DateFormatUtils @@ -45,4 +47,20 @@ object ECPathUtils { suffix + File.separator + ticketId } + def getECLogDirSuffix( + engineTypeLabel: EngineTypeLabel, + userCreatorLabel: UserCreatorLabel, + ticketId: String + ): String = { + if (null == engineTypeLabel || null == userCreatorLabel) { + return "" + } + val suffix = ECPathUtils.getECWOrkDirPathSuffix( + userCreatorLabel.getUser, + ticketId, + engineTypeLabel.getEngineType + ) + suffix + File.separator + "logs" + } + } diff --git a/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/utils/GovernanceConstant.scala b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/utils/GovernanceConstant.scala index 52e7802164..54927a84df 100644 --- a/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/utils/GovernanceConstant.scala +++ b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/utils/GovernanceConstant.scala @@ -31,5 +31,4 @@ object GovernanceConstant { val REQUEST_ENGINE_STATUS_BATCH_LIMIT = 500 - def RESULTSET_INDEX: String = "resultsetIndex" } diff --git a/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/utils/GovernanceUtils.scala b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/utils/GovernanceUtils.scala index 43fd598f71..ddcb17a3b2 100644 --- a/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/utils/GovernanceUtils.scala +++ b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/utils/GovernanceUtils.scala @@ -25,6 +25,7 @@ import org.apache.commons.lang3.StringUtils import java.io.File import java.util +import java.util.{ArrayList, List} object GovernanceUtils extends Logging { @@ -72,4 +73,52 @@ object GovernanceUtils extends Logging { } } + def killECProcessByPort(port: String, desc: String, isSudo: Boolean): Unit = { + val subProcessKillScriptPath = + Configuration.getLinkisHome() + "/sbin/kill-ec-process-by-port.sh" + if ( + StringUtils.isBlank(subProcessKillScriptPath) || !new File(subProcessKillScriptPath) + .exists() + ) { + logger.error(s"Failed to locate kill-script, $subProcessKillScriptPath not exist") + } else if (StringUtils.isNotBlank(port)) { + val cmd = if (isSudo) { + Array("sudo", "sh", subProcessKillScriptPath, port) + } else { + Array("sh", subProcessKillScriptPath, port) + } + logger.info( + s"Starting to kill process and sub-processes. desc: $desc Kill Command: " + cmd + .mkString(" ") + ) + + Utils.tryCatch { + val output = Utils.exec(cmd, 600 * 1000L) + logger.info(s"Kill Success! desc: $desc. msg:\n ${output}") + } { t => + logger.error(s"Kill error! desc: $desc.", t) + } + } + } + + /** + * find process id by port number + * @param processPort + * @return + */ + def findProcessIdentifier(processPort: String): String = { + val findCmd = + "sudo netstat -tunlp | grep :" + processPort + " | awk '{print $7}' | awk -F/ '{print $1}'" + val cmdList = new util.ArrayList[String] + cmdList.add("bash") + cmdList.add("-c") + cmdList.add(findCmd) + try Utils.exec(cmdList.toArray(new Array[String](0)), 5000L) + catch { + case e: Exception => + logger.warn("Method findPid failed, " + e.getMessage) + null + } + } + } diff --git a/linkis-computation-governance/linkis-computation-governance-common/src/test/scala/org/apache/linkis/governance/common/paser/ScalaCodeParserTest.scala b/linkis-computation-governance/linkis-computation-governance-common/src/test/scala/org/apache/linkis/governance/common/paser/ScalaCodeParserTest.scala index 3cfe787f61..04adf3446c 100644 --- a/linkis-computation-governance/linkis-computation-governance-common/src/test/scala/org/apache/linkis/governance/common/paser/ScalaCodeParserTest.scala +++ b/linkis-computation-governance/linkis-computation-governance-common/src/test/scala/org/apache/linkis/governance/common/paser/ScalaCodeParserTest.scala @@ -29,7 +29,7 @@ class ScalaCodeParserTest { "val codeBuffer = new ArrayBuffer[String]()\n val statementBuffer = new ArrayBuffer[String]()" val scalaCodeParser = new ScalaCodeParser val array = scalaCodeParser.parse(scalaCode) - Assertions.assertTrue(array.size == 1) + Assertions.assertTrue(array.size == 3) } @@ -41,7 +41,7 @@ class ScalaCodeParserTest { " def addInt( a:Int, b:Int )\n var sum:Int = 0\n sum = a + b\n return sum\n }" val scalaCodeParser = new ScalaCodeParser val array = scalaCodeParser.parse(abnormalCode) - Assertions.assertTrue(array.length == 1) + Assertions.assertTrue(array.length == 3) } @@ -54,7 +54,7 @@ class ScalaCodeParserTest { val scalaCodeParser = new ScalaCodeParser val array = scalaCodeParser.parse(importCode) - Assertions.assertTrue(array.length == 2) + Assertions.assertTrue(array.length == 4) } @@ -68,7 +68,7 @@ class ScalaCodeParserTest { val scalaCodeParser = new ScalaCodeParser val arrayResult1 = scalaCodeParser.parse(specialCodeExp1) - Assertions.assertTrue(arrayResult1.length == 2) + Assertions.assertTrue(arrayResult1.length == 4) val specialCodeExp2 = " @BeanProperty\n var id: Long = _\n @BeanProperty\n var status: Int = 0\n " + @@ -79,7 +79,7 @@ class ScalaCodeParserTest { ".append(data, that.data)\n .isEquals\n }" val arrayResult2 = scalaCodeParser.parse(specialCodeExp2) - Assertions.assertTrue(arrayResult2.length == 1) + Assertions.assertTrue(arrayResult2.length == 3) } diff --git a/linkis-computation-governance/linkis-computation-governance-common/src/test/scala/org/apache/linkis/governance/common/utils/GovernanceConstantTest.scala b/linkis-computation-governance/linkis-computation-governance-common/src/test/scala/org/apache/linkis/governance/common/utils/GovernanceConstantTest.scala index 891d43c8b7..22f3cee233 100644 --- a/linkis-computation-governance/linkis-computation-governance-common/src/test/scala/org/apache/linkis/governance/common/utils/GovernanceConstantTest.scala +++ b/linkis-computation-governance/linkis-computation-governance-common/src/test/scala/org/apache/linkis/governance/common/utils/GovernanceConstantTest.scala @@ -31,7 +31,6 @@ class GovernanceConstantTest { val taskresourceversionstr = GovernanceConstant.TASK_RESOURCE_VERSION_STR val taskresourcefilenamestr = GovernanceConstant.TASK_RESOURCE_FILE_NAME_STR val requestenginestatusbatchlimit = GovernanceConstant.REQUEST_ENGINE_STATUS_BATCH_LIMIT - val resultsetindex = GovernanceConstant.RESULTSET_INDEX Assertions.assertEquals("source", tasksourcemapkey) Assertions.assertEquals("resources", taskresourcesstr) @@ -39,7 +38,6 @@ class GovernanceConstantTest { Assertions.assertEquals("version", taskresourceversionstr) Assertions.assertEquals("fileName", taskresourcefilenamestr) Assertions.assertTrue(500 == requestenginestatusbatchlimit.intValue()) - Assertions.assertEquals("resultsetIndex", resultsetindex) } diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/org/apache/linkis/ecm/core/launch/ProcessEngineCommandBuilder.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/org/apache/linkis/ecm/core/launch/ProcessEngineCommandBuilder.scala index fade444fa0..4b5b1fab9e 100644 --- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/org/apache/linkis/ecm/core/launch/ProcessEngineCommandBuilder.scala +++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/org/apache/linkis/ecm/core/launch/ProcessEngineCommandBuilder.scala @@ -77,7 +77,7 @@ class UnixProcessEngineCommandBuilder extends ShellProcessEngineCommandBuilder { newLine("linkis_engineconn_errorcode=$?") newLine("if [ $linkis_engineconn_errorcode -ne 0 ]") newLine("then") - newLine(" tail -1000 ${LOG_DIRS}/stderr") + newLine(" timeout 10 tail -1000 ${LOG_DIRS}/stderr") newLine(" exit $linkis_engineconn_errorcode") newLine("fi") } diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/org/apache/linkis/ecm/core/launch/ProcessEngineConnLaunch.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/org/apache/linkis/ecm/core/launch/ProcessEngineConnLaunch.scala index 91aa93e5fc..cc79e24d4f 100644 --- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/org/apache/linkis/ecm/core/launch/ProcessEngineConnLaunch.scala +++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/org/apache/linkis/ecm/core/launch/ProcessEngineConnLaunch.scala @@ -166,8 +166,15 @@ trait ProcessEngineConnLaunch extends EngineConnLaunch with Logging { .findAvailPortByRange(GovernanceCommonConf.ENGINE_CONN_PORT_RANGE.getValue) .toString - var springConf = Map("server.port" -> engineConnPort, "spring.profiles.active" -> "engineconn") - + var springConf = + Map[String, String]("server.port" -> engineConnPort, "spring.profiles.active" -> "engineconn") + val properties = + PortUtils.readFromProperties(Configuration.getLinkisHome + "/conf/version.properties") + if (StringUtils.isNotBlank(properties.getProperty("version"))) { + springConf += ("eureka.instance.metadata-map.linkis.app.version" -> properties.getProperty( + "version" + )) + } request.creationDesc.properties.asScala.filter(_._1.startsWith("spring.")).foreach { case (k, v) => springConf = springConf + (k -> v) @@ -255,14 +262,15 @@ trait ProcessEngineConnLaunch extends EngineConnLaunch with Logging { } } + /** + * process exit code if process is null retur errorcode 10 + * @return + */ def processWaitFor: Int = { if (process != null) { process.waitFor } else { - throw new ECMCoreException( - CAN_NOT_GET_TERMINATED.getErrorCode, - CAN_NOT_GET_TERMINATED.getErrorDesc - ) + 10 } } diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/org/apache/linkis/ecm/core/report/NodeHealthReport.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/org/apache/linkis/ecm/core/report/NodeHealthReport.scala index 160025ed51..395c9258b8 100644 --- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/org/apache/linkis/ecm/core/report/NodeHealthReport.scala +++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/org/apache/linkis/ecm/core/report/NodeHealthReport.scala @@ -34,9 +34,9 @@ trait NodeHealthReport { def getNodeMsg: String - def getUsedResource: Resource +// def getUsedResource: Resource - def setUsedResource(resource: Resource): Unit +// def setUsedResource(resource: Resource): Unit def getTotalResource: Resource diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/org/apache/linkis/ecm/core/utils/PortUtils.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/org/apache/linkis/ecm/core/utils/PortUtils.scala index 4612467193..8552020493 100644 --- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/org/apache/linkis/ecm/core/utils/PortUtils.scala +++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/org/apache/linkis/ecm/core/utils/PortUtils.scala @@ -17,15 +17,16 @@ package org.apache.linkis.ecm.core.utils -import org.apache.linkis.common.utils.Utils +import org.apache.linkis.common.utils.{Logging, Utils} import org.apache.commons.io.IOUtils import org.apache.commons.lang3.StringUtils -import java.io.IOException +import java.io.{BufferedReader, FileReader, IOException} import java.net.ServerSocket +import java.util.Properties -object PortUtils { +object PortUtils extends Logging { /** * portRange: '-' is the separator @@ -62,4 +63,23 @@ object PortUtils { Utils.tryFinally(socket.getLocalPort)(IOUtils.closeQuietly(socket)) } + def readFromProperties(propertiesFile: String): Properties = { + val properties: Properties = new Properties + var reader: BufferedReader = null; + try { + reader = new BufferedReader(new FileReader(propertiesFile)) + properties.load(reader) + } catch { + case e: Exception => + logger.warn(s"loading vsersion faild with path $propertiesFile error:$e") + } finally { + try if (reader != null) reader.close + catch { + case e: Exception => + logger.warn(s"try to close buffered reader with error:${e.getMessage}") + } + } + properties + } + } diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/java/org/apache/linkis/ecm/server/service/impl/DefaultEngineConnKillService.java b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/java/org/apache/linkis/ecm/server/service/impl/DefaultEngineConnKillService.java new file mode 100644 index 0000000000..24d9792fb6 --- /dev/null +++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/java/org/apache/linkis/ecm/server/service/impl/DefaultEngineConnKillService.java @@ -0,0 +1,286 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.ecm.server.service.impl; + +import org.apache.linkis.common.ServiceInstance; +import org.apache.linkis.common.utils.Utils; +import org.apache.linkis.ecm.server.conf.ECMConfiguration; +import org.apache.linkis.ecm.server.service.EngineConnKillService; +import org.apache.linkis.engineconn.common.conf.EngineConnConf; +import org.apache.linkis.governance.common.utils.GovernanceUtils; +import org.apache.linkis.manager.common.constant.AMConstant; +import org.apache.linkis.manager.common.protocol.engine.EngineStopRequest; +import org.apache.linkis.manager.common.protocol.engine.EngineStopResponse; +import org.apache.linkis.manager.common.protocol.engine.EngineSuicideRequest; +import org.apache.linkis.rpc.Sender; +import org.apache.linkis.rpc.message.annotation.Receiver; + +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.StringUtils; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileReader; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class DefaultEngineConnKillService implements EngineConnKillService { + + private static final Logger logger = LoggerFactory.getLogger(DefaultEngineConnKillService.class); + + private static final ThreadPoolExecutor ecYarnAppKillService = + Utils.newCachedThreadPool(10, "ECM-Kill-EC-Yarn-App", true); + + @Override + @Receiver + public EngineStopResponse dealEngineConnStop(EngineStopRequest engineStopRequest) { + logger.info("received EngineStopRequest " + engineStopRequest); + String pid = null; + if (AMConstant.PROCESS_MARK.equals(engineStopRequest.getIdentifierType()) + && StringUtils.isNotBlank(engineStopRequest.getIdentifier())) { + pid = engineStopRequest.getIdentifier(); + } + logger.info("dealEngineConnStop return pid: {}", pid); + EngineStopResponse response = new EngineStopResponse(); + if (StringUtils.isNotBlank(pid)) { + if (!killEngineConnByPid(pid, engineStopRequest.getServiceInstance())) { + response.setStopStatus(false); + response.setMsg( + "Kill engine " + engineStopRequest.getServiceInstance().toString() + " failed."); + } else { + response.setStopStatus(true); + response.setMsg( + "Kill engine " + engineStopRequest.getServiceInstance().toString() + " succeed."); + } + } else { + String processPort = engineStopRequest.getServiceInstance().getInstance().split(":")[1]; + logger.warn("Kill EC {} by port {}", engineStopRequest.getServiceInstance(), processPort); + if (!killEngineConnByPort(processPort, engineStopRequest.getServiceInstance())) { + response.setStopStatus(false); + response.setMsg( + "Kill engine " + engineStopRequest.getServiceInstance().toString() + " failed."); + } else { + response.setStopStatus(true); + response.setMsg( + "Kill engine " + engineStopRequest.getServiceInstance().toString() + " succeed."); + } + } + + // Requires default kill yarn appid + if (AMConstant.PROCESS_MARK.equals(engineStopRequest.getIdentifierType())) { + killYarnAppIdOfOneEc(engineStopRequest); + } + + if (!response.getStopStatus()) { + EngineSuicideRequest request = + new EngineSuicideRequest( + engineStopRequest.getServiceInstance(), engineStopRequest.getUser()); + try { + Sender.getSender(engineStopRequest.getServiceInstance()).send(request); + response.setStopStatus(true); + response.setMsg(response.getMsg() + " Now send suicide request to engine."); + } catch (Exception e) { + response.setMsg( + response.getMsg() + " Sended suicide request to engine error, " + e.getMessage()); + } + } + return response; + } + + public void killYarnAppIdOfOneEc(EngineStopRequest engineStopRequest) { + String logDirSuffix = engineStopRequest.getLogDirSuffix(); + ServiceInstance serviceInstance = engineStopRequest.getServiceInstance(); + String engineType = engineStopRequest.getEngineType(); + String engineConnInstance = serviceInstance.toString(); + String engineLogDir; + if (logDirSuffix.startsWith(ECMConfiguration.ENGINECONN_ROOT_DIR())) { + engineLogDir = logDirSuffix; + } else { + engineLogDir = ECMConfiguration.ENGINECONN_ROOT_DIR() + File.separator + logDirSuffix; + } + logger.info( + "try to kill yarn app ids in the engine of: [{}] engineLogDir: [{}]", + engineConnInstance, + engineLogDir); + final String errEngineLogPath = engineLogDir.concat(File.separator).concat("yarnApp"); + logger.info( + "try to parse the yarn app id from the engine err log file path: [{}]", errEngineLogPath); + File file = new File(errEngineLogPath); + if (file.exists()) { + ecYarnAppKillService.execute( + () -> { + BufferedReader in = null; + try { + in = new BufferedReader(new FileReader(errEngineLogPath)); + String line; + String regex = getYarnAppRegexByEngineType(engineType); + if (StringUtils.isBlank(regex)) { + return; + } + Pattern pattern = Pattern.compile(regex); + List appIds = new ArrayList<>(); + while ((line = in.readLine()) != null) { + if (StringUtils.isNotBlank(line)) { + Matcher mApp = pattern.matcher(line); + if (mApp.find()) { + String candidate1 = mApp.group(mApp.groupCount()); + if (!appIds.contains(candidate1)) { + appIds.add(candidate1); + } + } + } + } + GovernanceUtils.killYarnJobApp(appIds); + logger.info("finished kill yarn app ids in the engine of ({}).", engineConnInstance); + } catch (IOException ioEx) { + if (ioEx instanceof FileNotFoundException) { + logger.error("the engine log file {} not found.", errEngineLogPath); + } else { + logger.error( + "the engine log file parse failed. the reason is {}", ioEx.getMessage()); + } + } finally { + IOUtils.closeQuietly(in); + } + }); + } + } + + private String getYarnAppRegexByEngineType(String engineType) { + if (StringUtils.isBlank(engineType)) { + return ""; + } + String regex; + switch (engineType) { + case "spark": + case "shell": + regex = EngineConnConf.SPARK_ENGINE_CONN_YARN_APP_ID_PARSE_REGEX().getValue(); + break; + case "sqoop": + regex = EngineConnConf.SQOOP_ENGINE_CONN_YARN_APP_ID_PARSE_REGEX().getValue(); + break; + case "flink": + case "hive": + regex = EngineConnConf.HIVE_ENGINE_CONN_YARN_APP_ID_PARSE_REGEX().getValue(); + break; + default: + regex = ""; + } + return regex; + } + + private boolean killEngineConnByPid(String processId, ServiceInstance serviceInstance) { + logger.info("try to kill {} toString with pid({}).", serviceInstance.toString(), processId); + if (StringUtils.isNotBlank(processId)) { + if (ECMConfiguration.ECM_PROCESS_SCRIPT_KILL()) { + GovernanceUtils.killProcess(processId, serviceInstance.toString(), true); + } else { + killProcessByKillCmd(processId, serviceInstance.toString()); + } + return !isProcessAlive(processId); + } else { + logger.warn("cannot kill {} with empty pid.", serviceInstance); + return false; + } + } + + private boolean killEngineConnByPort(String port, ServiceInstance serviceInstance) { + logger.info("try to kill {} toString with port({}).", serviceInstance.toString(), port); + if (StringUtils.isNotBlank(port)) { + GovernanceUtils.killECProcessByPort(port, serviceInstance.toString(), true); + return !isProcessAliveByPort(port); + } else { + logger.warn("cannot kill {} with empty port.", serviceInstance); + return false; + } + } + + private boolean isProcessAlive(String pid) { + String findCmd = + "ps -ef | grep " + + pid + + " | grep EngineConnServer | awk '{print \"exists_\"$2}' | grep " + + pid + + "|| true"; + List cmdList = new ArrayList<>(); + cmdList.add("bash"); + cmdList.add("-c"); + cmdList.add(findCmd); + try { + String rs = Utils.exec(cmdList.toArray(new String[0]), 5000L); + return null != rs && rs.contains("exists_" + pid); + } catch (Exception e) { + logger.warn("Method isProcessAlive failed", e); + return false; + } + } + + private boolean isProcessAliveByPort(String port) { + String findCmd = + "ps -ef | grep server.port= " + + port + + " | grep EngineConnServer | awk -F \"server.port=\" '{print \"exists_\"$2}'"; + List cmdList = new ArrayList<>(); + cmdList.add("bash"); + cmdList.add("-c"); + cmdList.add(findCmd); + try { + String rs = Utils.exec(cmdList.toArray(new String[0]), 5000L); + return null != rs && rs.contains("exists_" + port); + } catch (Exception e) { + logger.warn("Method isProcessAlive failed", e); + return false; + } + } + + private void killProcessByKillCmd(String pid, String desc) { + String k15cmd = "sudo kill " + pid; + String k9cmd = "sudo kill -9 " + pid; + int tryNum = 0; + try { + while (isProcessAlive(pid) && tryNum <= 3) { + logger.info( + "{} still alive with pid({}), use shell command to kill it. try {}++", + desc, + pid, + tryNum++); + if (tryNum <= 3) { + Utils.exec(k15cmd.split(" "), 3000L); + } else { + logger.info( + "{} still alive with pid({}). try {}, use shell command to kill -9 it", + desc, + pid, + tryNum); + Utils.exec(k9cmd.split(" "), 3000L); + } + Thread.sleep(5000); + } + } catch (InterruptedException e) { + logger.error("Interrupted while killing engine {} with pid({})." + desc, pid); + } + } +} diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/listener/ECMReadyEvent.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/listener/ECMReadyEvent.scala index 6bcb6c3b8b..db4ccea7f9 100644 --- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/listener/ECMReadyEvent.scala +++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/listener/ECMReadyEvent.scala @@ -19,22 +19,11 @@ package org.apache.linkis.ecm.server.listener import org.apache.linkis.ecm.core.engineconn.EngineConn import org.apache.linkis.ecm.core.listener.ECMEvent -import org.apache.linkis.governance.common.protocol.task.ResponseEngineConnPid -import org.apache.linkis.manager.common.entity.enumeration.NodeStatus -import org.apache.linkis.protocol.callback.{YarnAPPIdCallbackProtocol, YarnInfoCallbackProtocol} +import org.apache.linkis.manager.common.protocol.engine.EngineStopRequest case class ECMReadyEvent(params: Array[String]) extends ECMEvent case class ECMClosedEvent() extends ECMEvent -case class EngineConnStatusChageEvent(from: NodeStatus, to: NodeStatus) extends ECMEvent - -case class YarnAppIdCallbackEvent(protocol: YarnAPPIdCallbackProtocol) extends ECMEvent - -case class YarnInfoCallbackEvent(protocol: YarnInfoCallbackProtocol) extends ECMEvent - -case class EngineConnPidCallbackEvent(protocol: ResponseEngineConnPid) extends ECMEvent - -case class EngineConnAddEvent(conn: EngineConn) extends ECMEvent - -case class EngineConnStatusChangeEvent(tickedId: String, updateStatus: NodeStatus) extends ECMEvent +case class EngineConnStopEvent(engineConn: EngineConn, engineStopRequest: EngineStopRequest) + extends ECMEvent diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/operator/EngineConnLogOperator.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/operator/EngineConnLogOperator.scala index bc856ba681..66327dadcf 100644 --- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/operator/EngineConnLogOperator.scala +++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/operator/EngineConnLogOperator.scala @@ -17,15 +17,13 @@ package org.apache.linkis.ecm.server.operator -import org.apache.linkis.DataWorkCloudApplication import org.apache.linkis.common.conf.CommonVars import org.apache.linkis.common.utils.{Logging, Utils} import org.apache.linkis.ecm.errorcode.EngineconnServerErrorCodeSummary._ import org.apache.linkis.ecm.server.conf.ECMConfiguration import org.apache.linkis.ecm.server.exception.ECMErrorException -import org.apache.linkis.ecm.server.service.{EngineConnListService, LocalDirsHandleService} +import org.apache.linkis.ecm.server.service.LocalDirsHandleService import org.apache.linkis.manager.common.operator.Operator -import org.apache.linkis.manager.common.protocol.em.ECMOperateRequest import org.apache.commons.io.IOUtils import org.apache.commons.io.input.ReversedLinesFileReader @@ -37,14 +35,10 @@ import java.text.MessageFormat import java.util import java.util.Collections -import scala.collection.JavaConverters.asScalaBufferConverter import scala.util.matching.Regex class EngineConnLogOperator extends Operator with Logging { - private var engineConnListService: EngineConnListService = _ - private var localDirsHandleService: LocalDirsHandleService = _ - override def getNames: Array[String] = Array(EngineConnLogOperator.OPERATOR_NAME) override def apply(implicit parameters: Map[String, Any]): Map[String, Any] = { @@ -95,10 +89,7 @@ class EngineConnLogOperator extends Operator with Logging { var readLine, skippedLine, lineNum = 0 var rowIgnore = false var ignoreLine = 0 - val linePattern = Option(EngineConnLogOperator.MULTILINE_PATTERN.getValue) match { - case Some(pattern) => pattern.r - case _ => null - } + val linePattern = getLinePattern val maxMultiline = EngineConnLogOperator.MULTILINE_MAX.getValue Utils.tryFinally { var line = randomAndReversedReadLine() @@ -155,59 +146,20 @@ class EngineConnLogOperator extends Operator with Logging { logPath } + protected def getLinePattern: Regex = { + Option(EngineConnLogOperator.MULTILINE_PATTERN.getValue) match { + case Some(pattern) => pattern.r + case _ => null + } + } + protected def getEngineConnInfo(implicit parameters: Map[String, Any] ): (String, String, String) = { - if (engineConnListService == null) { - engineConnListService = - DataWorkCloudApplication.getApplicationContext.getBean(classOf[EngineConnListService]) - localDirsHandleService = - DataWorkCloudApplication.getApplicationContext.getBean(classOf[LocalDirsHandleService]) - } val logDIrSuffix = getAs("logDirSuffix", "") - val (engineConnLogDir, engineConnInstance, ticketId) = - if (StringUtils.isNotBlank(logDIrSuffix)) { - val ecLogPath = ECMConfiguration.ENGINECONN_ROOT_DIR + File.separator + logDIrSuffix - val ticketId = getAs("ticketId", "") - (ecLogPath, "", ticketId) - } else { - val engineConnInstance = getAs( - ECMOperateRequest.ENGINE_CONN_INSTANCE_KEY, - getAs[String]("engineConnInstance", null) - ) - Option(engineConnInstance) - .flatMap { instance => - engineConnListService.getEngineConns.asScala.find( - _.getServiceInstance.getInstance == instance - ) - } - .map(engineConn => - ( - engineConn.getEngineConnManagerEnv.engineConnLogDirs, - engineConnInstance, - engineConn.getTickedId - ) - ) - .getOrElse { - val ticketId = getAs("ticketId", "") - if (StringUtils.isBlank(ticketId)) { - throw new ECMErrorException( - BOTH_NOT_EXISTS.getErrorCode, - s"the parameters of ${ECMOperateRequest.ENGINE_CONN_INSTANCE_KEY}, engineConnInstance and ticketId are both not exists." - ) - } - val logDir = engineConnListService - .getEngineConn(ticketId) - .map(_.getEngineConnManagerEnv.engineConnLogDirs) - .getOrElse { - val creator = getAsThrow[String]("creator") - val engineConnType = getAsThrow[String]("engineConnType") - localDirsHandleService.getEngineConnLogDir(creator, ticketId, engineConnType) - } - (logDir, engineConnInstance, ticketId) - } - } - (ticketId, engineConnInstance, engineConnLogDir) + val ecLogPath = ECMConfiguration.ENGINECONN_ROOT_DIR + File.separator + logDIrSuffix + val ticketId = getAs("ticketId", "") + (ticketId, "", ecLogPath) } private def includeLine( diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/operator/EngineConnYarnLogOperator.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/operator/EngineConnYarnLogOperator.scala index e00d16b519..36e7ddfc5f 100644 --- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/operator/EngineConnYarnLogOperator.scala +++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/operator/EngineConnYarnLogOperator.scala @@ -17,20 +17,36 @@ package org.apache.linkis.ecm.server.operator -import org.apache.linkis.common.exception.LinkisCommonErrorException +import org.apache.linkis.common.conf.CommonVars +import org.apache.linkis.common.io.FsPath import org.apache.linkis.common.utils.Utils -import org.apache.linkis.ecm.core.conf.ECMErrorCode import org.apache.linkis.ecm.errorcode.EngineconnServerErrorCodeSummary._ import org.apache.linkis.ecm.server.exception.ECMErrorException +import org.apache.linkis.storage.FSFactory +import org.apache.linkis.storage.fs.FileSystem +import org.apache.linkis.storage.utils.StorageUtils + +import org.apache.commons.lang3.StringUtils import java.io.File import java.text.MessageFormat -import java.util.concurrent.TimeUnit +import java.util +import java.util.concurrent.{Callable, ConcurrentHashMap, ExecutorService, Future, TimeUnit} import scala.collection.JavaConverters._ +import scala.util.matching.Regex class EngineConnYarnLogOperator extends EngineConnLogOperator { + private implicit val fs: FileSystem = + FSFactory.getFs(StorageUtils.FILE).asInstanceOf[FileSystem] + + /** + * Yarn log fetchers + */ + private def yarnLogFetchers: ConcurrentHashMap[String, Future[String]] = + new ConcurrentHashMap[String, Future[String]]() + override def getNames: Array[String] = Array(EngineConnYarnLogOperator.OPERATOR_NAME) override def apply(implicit parameters: Map[String, Any]): Map[String, Any] = { @@ -38,75 +54,118 @@ class EngineConnYarnLogOperator extends EngineConnLogOperator { Utils.tryFinally { result = super.apply(parameters) result - } { - result.get("logPath") match { - case Some(path: String) => - val logFile = new File(path) - if (logFile.exists() && logFile.getName.startsWith(".")) { - // If is a temporary file, drop it - logger.info(s"Delete the temporary yarn log file: [$path]") - if (!logFile.delete()) { - logger.warn(s"Fail to delete the temporary yarn log file: [$path]") - } - } - } - } + } {} } override def getLogPath(implicit parameters: Map[String, Any]): File = { + val applicationId = getAsThrow[String]("yarnApplicationId") val (ticketId, engineConnInstance, engineConnLogDir) = getEngineConnInfo(parameters) - val rootLogDir = new File(engineConnLogDir) - if (!rootLogDir.exists() || !rootLogDir.isDirectory) { + val rootLogPath = EngineConnYarnLogOperator.YARN_LOG_STORAGE_PATH.getValue match { + case storePath if StringUtils.isNotBlank(storePath) => + val logPath = new FsPath(StorageUtils.FILE_SCHEMA + storePath + "/" + applicationId) + // Invoke to create directory + fs.mkdir(logPath) + // 777 permission + fs.setPermission(logPath, "rwxrwxrwx") + logPath + case _ => new FsPath(StorageUtils.FILE_SCHEMA + engineConnLogDir) + } + if (!fs.exists(rootLogPath) || !rootLogPath.toFile.isDirectory) { throw new ECMErrorException( LOG_IS_NOT_EXISTS.getErrorCode, - MessageFormat.format(LOG_IS_NOT_EXISTS.getErrorDesc, rootLogDir) + MessageFormat.format(LOG_IS_NOT_EXISTS.getErrorDesc, rootLogPath.getPath) ) } val creator = getAsThrow[String]("creator") - val applicationId = getAsThrow[String]("yarnApplicationId") - var logPath = new File(engineConnLogDir, "yarn_" + applicationId) - if (!logPath.exists()) { - val tempLogFile = - s".yarn_${applicationId}_${System.currentTimeMillis()}_${Thread.currentThread().getId}" - Utils.tryCatch { - var command = s"yarn logs -applicationId $applicationId >> $rootLogDir/$tempLogFile" - logger.info(s"Fetch yarn logs to temporary file: [$command]") - val processBuilder = new ProcessBuilder(sudoCommands(creator, command): _*) - processBuilder.environment.putAll(sys.env.asJava) - processBuilder.redirectErrorStream(false) - val process = processBuilder.start() - val waitFor = process.waitFor(5, TimeUnit.SECONDS) - logger.trace(s"waitFor: ${waitFor}, result: ${process.exitValue()}") - if (waitFor && process.waitFor() == 0) { - command = s"mv $rootLogDir/$tempLogFile $rootLogDir/yarn_$applicationId" - logger.info(s"Move and save yarn logs: [$command]") - Utils.exec(sudoCommands(creator, command)) - } else { - logPath = new File(engineConnLogDir, tempLogFile) - if (!logPath.exists()) { - throw new LinkisCommonErrorException( - -1, - s"Fetch yarn logs timeout, log aggregation has not completed or is not enabled" - ) - } + var logPath = new FsPath( + StorageUtils.FILE_SCHEMA + rootLogPath.getPath + "/yarn_" + applicationId + ) + if (!fs.exists(logPath)) { + val fetcher = yarnLogFetchers.computeIfAbsent( + applicationId, + new util.function.Function[String, Future[String]] { + override def apply(v1: String): Future[String] = + requestToFetchYarnLogs(creator, applicationId, rootLogPath.getPath) } - } { case e: Exception => - throw new LinkisCommonErrorException( - -1, - s"Fail to fetch yarn logs application: $applicationId, message: ${e.getMessage}" - ) + ) + // Just wait 5 seconds + Option(fetcher.get(5, TimeUnit.SECONDS)) match { + case Some(path) => logPath = new FsPath(StorageUtils.FILE_SCHEMA + path) + case _ => } + } - if (!logPath.exists() || !logPath.isFile) { + if (!fs.exists(logPath) || logPath.toFile.isDirectory) { throw new ECMErrorException( LOGFILE_IS_NOT_EXISTS.getErrorCode, - MessageFormat.format(LOGFILE_IS_NOT_EXISTS.getErrorDesc, logPath) + MessageFormat.format(LOGFILE_IS_NOT_EXISTS.getErrorDesc, logPath.getPath) ) } logger.info( s"Try to fetch EngineConn(id: $ticketId, instance: $engineConnInstance) yarn logs from ${logPath.getPath} in application id: $applicationId" ) - logPath + logPath.toFile + } + + /** + * Not support line pattern in yarn log + * @return + */ + override protected def getLinePattern: Regex = null + + /** + * Request the log fetcher + * + * @param creator + * creator + * @param applicationId + * application id + * @param logPath + * log path + * @return + */ + private def requestToFetchYarnLogs( + creator: String, + applicationId: String, + yarnLogDir: String + ): Future[String] = { + EngineConnYarnLogOperator.YARN_LOG_FETCH_SCHEDULER.submit(new Callable[String] { + override def call(): String = { + val logPath = new FsPath(StorageUtils.FILE_SCHEMA + yarnLogDir + "/yarn_" + applicationId) + if (!fs.exists(logPath)) { + val tempLogFile = + s".yarn_${applicationId}_${System.currentTimeMillis()}_${Thread.currentThread().getId}" + Utils.tryCatch { + var command = + s"yarn logs -applicationId $applicationId >> $yarnLogDir/$tempLogFile" + logger.info(s"Fetch yarn logs to temporary file: [$command]") + val processBuilder = new ProcessBuilder(sudoCommands(creator, command): _*) + processBuilder.environment.putAll(sys.env.asJava) + processBuilder.redirectErrorStream(false) + val process = processBuilder.start() + val exitCode = process.waitFor() + logger.trace(s"Finish to fetch yan logs to temporary file, result: ${exitCode}") + if (exitCode == 0) { + command = s"mv $yarnLogDir/$tempLogFile $yarnLogDir/yarn_$applicationId" + logger.info(s"Move and save yarn logs(${applicationId}): [$command]") + Utils.exec(sudoCommands(creator, command)) + } + } { e: Throwable => + logger.error( + s"Fail to fetch yarn logs application: $applicationId, message: ${e.getMessage}" + ) + } + val tmpFile = new File(yarnLogDir, tempLogFile) + if (tmpFile.exists()) { + logger.info(s"Delete temporary file: [${tempLogFile}] in yarn logs fetcher") + tmpFile.delete() + } + } + // Remove future + yarnLogFetchers.remove(applicationId) + if (fs.exists(logPath)) logPath.getPath else null + } + }) } private def sudoCommands(creator: String, command: String): Array[String] = { @@ -121,4 +180,15 @@ class EngineConnYarnLogOperator extends EngineConnLogOperator { object EngineConnYarnLogOperator { val OPERATOR_NAME = "engineConnYarnLog" + + // Specific the path to store the yarn logs + val YARN_LOG_STORAGE_PATH: CommonVars[String] = + CommonVars("linkis.engineconn.log.yarn.storage-path", "") + + val YARN_LOG_FETCH_THREAD: CommonVars[Int] = + CommonVars("linkis.engineconn.log.yarn.fetch.thread-num", 5) + + val YARN_LOG_FETCH_SCHEDULER: ExecutorService = + Utils.newFixedThreadPool(YARN_LOG_FETCH_THREAD.getValue + 1, "yarn_logs_fetch", false) + } diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/report/DefaultECMHealthReport.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/report/DefaultECMHealthReport.scala index 6fdf8b3488..150d0be6ba 100644 --- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/report/DefaultECMHealthReport.scala +++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/report/DefaultECMHealthReport.scala @@ -69,9 +69,9 @@ class DefaultECMHealthReport extends ECMHealthReport { override def getNodeMsg: String = nodeMsg - override def getUsedResource: Resource = usedResource +// override def getUsedResource: Resource = usedResource - override def setUsedResource(usedResource: Resource): Unit = this.usedResource = usedResource +// override def setUsedResource(usedResource: Resource): Unit = this.usedResource = usedResource override def getTotalResource: Resource = totalResource diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/EngineConnListService.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/EngineConnListService.scala deleted file mode 100644 index d36d35cceb..0000000000 --- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/EngineConnListService.scala +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.ecm.server.service - -import org.apache.linkis.ecm.core.engineconn.EngineConn -import org.apache.linkis.ecm.core.launch.EngineConnLaunchRunner -import org.apache.linkis.manager.common.entity.resource.Resource - -import java.util - -/** - * The enginelistservice interface manages the interface started by the engine The most important - * submit method is to put the thread that starts the engine into the thread pool to start - * EngineListService接口管理引擎启动的接口 最重要的submit方法是将启动引擎的线程放入到线程池中进行启动 - */ -trait EngineConnListService { - - def init(): Unit - - def getEngineConn(engineConnId: String): Option[EngineConn] - - def getEngineConns: util.List[EngineConn] - - def addEngineConn(engineConn: EngineConn): Unit - - def killEngineConn(engineConnId: String): Unit - - def getUsedResources: Resource - - def submit(runner: EngineConnLaunchRunner): Option[EngineConn] - -} diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/AbstractEngineConnLaunchService.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/AbstractEngineConnLaunchService.scala index 6ac10d1e14..49e75fb6b8 100644 --- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/AbstractEngineConnLaunchService.scala +++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/AbstractEngineConnLaunchService.scala @@ -25,22 +25,25 @@ import org.apache.linkis.ecm.server.LinkisECMApplication import org.apache.linkis.ecm.server.conf.ECMConfiguration._ import org.apache.linkis.ecm.server.engineConn.DefaultEngineConn import org.apache.linkis.ecm.server.hook.ECMHook -import org.apache.linkis.ecm.server.listener.{EngineConnAddEvent, EngineConnStatusChangeEvent} +import org.apache.linkis.ecm.server.listener.EngineConnStopEvent import org.apache.linkis.ecm.server.service.{EngineConnLaunchService, ResourceLocalizationService} import org.apache.linkis.ecm.server.util.ECMUtils import org.apache.linkis.governance.common.conf.GovernanceCommonConf -import org.apache.linkis.governance.common.utils.JobUtils +import org.apache.linkis.governance.common.utils.{ECPathUtils, JobUtils, LoggerUtils} +import org.apache.linkis.manager.common.constant.AMConstant import org.apache.linkis.manager.common.entity.enumeration.NodeStatus -import org.apache.linkis.manager.common.entity.enumeration.NodeStatus.Failed import org.apache.linkis.manager.common.entity.node.{AMEngineNode, EngineNode} -import org.apache.linkis.manager.common.protocol.engine.EngineConnStatusCallbackToAM +import org.apache.linkis.manager.common.protocol.engine.{ + EngineConnStatusCallbackToAM, + EngineStopRequest +} import org.apache.linkis.manager.engineplugin.common.launch.entity.EngineConnLaunchRequest +import org.apache.linkis.manager.label.utils.LabelUtil import org.apache.linkis.rpc.Sender import org.apache.commons.lang3.exception.ExceptionUtils -import scala.concurrent.{ExecutionContextExecutorService, Future} -import scala.util.{Failure, Success} +import scala.concurrent.ExecutionContextExecutorService abstract class AbstractEngineConnLaunchService extends EngineConnLaunchService with Logging { @@ -61,8 +64,9 @@ abstract class AbstractEngineConnLaunchService extends EngineConnLaunchService w } override def launchEngineConn(request: EngineConnLaunchRequest, duration: Long): EngineNode = { - // 1.创建engineConn和runner,launch 并设置基础属性 + // create engineConn/runner/launch val taskId = JobUtils.getJobIdFromStringMap(request.creationDesc.properties) + LoggerUtils.setJobIdMDC(taskId) logger.info("TaskId: {} try to launch a new EngineConn with {}.", taskId: Any, request: Any) val conn = createEngineConn val runner = createEngineConnLaunchRunner @@ -77,11 +81,9 @@ abstract class AbstractEngineConnLaunchService extends EngineConnLaunchService w conn.setStatus(NodeStatus.Starting) conn.setEngineConnInfo(new EngineConnInfo) conn.setEngineConnManagerEnv(launch.getEngineConnManagerEnv()) - // 2.资源本地化，并且设置ecm的env环境信息 + // get ec Resource getResourceLocalizationServie.handleInitEngineConnResources(request, conn) - // 3.添加到list - LinkisECMApplication.getContext.getECMSyncListenerBus.postToAll(EngineConnAddEvent(conn)) - // 4.run + // start ec Utils.tryCatch { beforeLaunch(request, conn, duration) runner.run() @@ -95,44 +97,12 @@ abstract class AbstractEngineConnLaunchService extends EngineConnLaunchService w case _ => } afterLaunch(request, conn, duration) - - val future = Future { - logger.info( - "TaskId: {} with request {} wait engineConn {} start", - Array(taskId, request, conn.getServiceInstance): _* - ) - waitEngineConnStart(request, conn, duration) - } - - future onComplete { - case Failure(t) => - logger.error( - "TaskId: {} init {} failed. {} with request {}", - Array( - taskId, - conn.getServiceInstance, - conn.getEngineConnLaunchRunner.getEngineConnLaunch - .getEngineConnManagerEnv() - .engineConnWorkDir, - request - ): _* - ) - LinkisECMApplication.getContext.getECMSyncListenerBus.postToAll( - EngineConnStatusChangeEvent(conn.getTickedId, Failed) - ) - case Success(_) => - logger.info( - "TaskId: {} init {} succeed. {} with request {}", - Array( - taskId, - conn.getServiceInstance, - conn.getEngineConnLaunchRunner.getEngineConnLaunch - .getEngineConnManagerEnv() - .engineConnWorkDir, - request - ): _* - ) - } + logger.info( + "TaskId: {} with request {} wait engineConn {} start", + Array(taskId, request, conn.getServiceInstance): _* + ) + // start ec monitor thread + startEngineConnMonitorStart(request, conn) } { t => logger.error( "TaskId: {} init {} failed, {}, with request {} now stop and delete it. message: {}", @@ -147,31 +117,43 @@ abstract class AbstractEngineConnLaunchService extends EngineConnLaunchService w t ): _* ) - conn.getEngineConnLaunchRunner.stop() Sender .getSender(MANAGER_SERVICE_NAME) .send( EngineConnStatusCallbackToAM( conn.getServiceInstance, - NodeStatus.ShuttingDown, + NodeStatus.Failed, " wait init failed , reason " + ExceptionUtils.getRootCauseMessage(t) ) ) - LinkisECMApplication.getContext.getECMSyncListenerBus.postToAll( - EngineConnStatusChangeEvent(conn.getTickedId, Failed) + conn.setStatus(NodeStatus.Failed) + val engineType = LabelUtil.getEngineType(request.labels) + val logPath = Utils.tryCatch(conn.getEngineConnManagerEnv.engineConnLogDirs) { t => + ECPathUtils.getECWOrkDirPathSuffix(request.user, request.ticketId, engineType) + } + val engineStopRequest = new EngineStopRequest + engineStopRequest.setEngineType(engineType) + engineStopRequest.setUser(request.user) + engineStopRequest.setIdentifier(conn.getPid) + engineStopRequest.setIdentifierType(AMConstant.PROCESS_MARK) + engineStopRequest.setLogDirSuffix(logPath) + engineStopRequest.setServiceInstance(conn.getServiceInstance) + LinkisECMApplication.getContext.getECMAsyncListenerBus.post( + EngineConnStopEvent(conn, engineStopRequest) ) + LoggerUtils.removeJobIdMDC() throw t } + LoggerUtils.removeJobIdMDC() val engineNode = new AMEngineNode() engineNode.setLabels(conn.getLabels) - engineNode.setServiceInstance(conn.getServiceInstance) engineNode.setOwner(request.user) - engineNode.setMark("process") + engineNode.setMark(AMConstant.PROCESS_MARK) engineNode } - def waitEngineConnStart(request: EngineConnLaunchRequest, conn: EngineConn, duration: Long): Unit + def startEngineConnMonitorStart(request: EngineConnLaunchRequest, conn: EngineConn): Unit def createEngineConn: EngineConn = new DefaultEngineConn diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultECMHealthService.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultECMHealthService.scala index 132749cbec..be879f6877 100644 --- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultECMHealthService.scala +++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultECMHealthService.scala @@ -25,7 +25,7 @@ import org.apache.linkis.ecm.server.conf.ECMConfiguration import org.apache.linkis.ecm.server.conf.ECMConfiguration._ import org.apache.linkis.ecm.server.listener.{ECMClosedEvent, ECMReadyEvent} import org.apache.linkis.ecm.server.report.DefaultECMHealthReport -import org.apache.linkis.ecm.server.service.{ECMHealthService, EngineConnListService} +import org.apache.linkis.ecm.server.service.ECMHealthService import org.apache.linkis.ecm.server.util.ECMUtils import org.apache.linkis.manager.common.entity.enumeration.{NodeHealthy, NodeStatus} import org.apache.linkis.manager.common.entity.metrics.{NodeHealthyInfo, NodeOverLoadInfo} @@ -38,8 +38,6 @@ import org.apache.linkis.manager.common.protocol.node.{ import org.apache.linkis.rpc.Sender import org.apache.linkis.rpc.message.annotation.Receiver -import org.springframework.beans.factory.annotation.Autowired - import java.util.Date import java.util.concurrent.TimeUnit @@ -79,9 +77,6 @@ class DefaultECMHealthService extends ECMHealthService with ECMEventListener { TimeUnit.SECONDS ) - @Autowired - private var engineConnListService: EngineConnListService = _ - override def getLastEMHealthReport: ECMHealthReport = { val report = new DefaultECMHealthReport report.setNodeId(LinkisECMApplication.getECMServiceInstance.toString) @@ -89,7 +84,6 @@ class DefaultECMHealthService extends ECMHealthService with ECMEventListener { // todo report right metrics report.setTotalResource(maxResource) report.setProtectedResource(minResource) - report.setUsedResource(engineConnListService.getUsedResources) report.setReportTime(new Date().getTime) report.setRunningEngineConns( LinkisECMApplication.getContext.getECMMetrics.getRunningEngineConns @@ -117,7 +111,6 @@ class DefaultECMHealthService extends ECMHealthService with ECMEventListener { // todo report latest engineconn metrics resource.setMaxResource(maxResource) resource.setMinResource(minResource) - resource.setUsedResource(report.getUsedResource) heartbeat.setNodeResource(resource) heartbeat.setHeartBeatMsg("") val nodeHealthyInfo = new NodeHealthyInfo diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultECMRegisterService.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultECMRegisterService.scala index eb9206c963..4c7807dad1 100644 --- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultECMRegisterService.scala +++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultECMRegisterService.scala @@ -37,7 +37,9 @@ import java.util.Collections class DefaultECMRegisterService extends ECMRegisterService with ECMEventListener with Logging { - private implicit def readyEvent2RegisterECMRequest(event: ECMReadyEvent): RegisterEMRequest = { + private var unRegisterFlag = false + + private def readyEvent2RegisterECMRequest(event: ECMReadyEvent): RegisterEMRequest = { val request = new RegisterEMRequest val instance = Sender.getThisServiceInstance request.setUser(Utils.getJvmUser) @@ -50,14 +52,11 @@ class DefaultECMRegisterService extends ECMRegisterService with ECMEventListener private def getLabelsFromArgs(params: Array[String]): util.Map[String, AnyRef] = { import scala.collection.JavaConverters._ - val labelRegex = """label\.(.+)\.(.+)=(.+)""".r val labels = new util.HashMap[String, AnyRef]() - // TODO: magic labels.asScala += LabelKeyConstant.SERVER_ALIAS_KEY -> Collections.singletonMap( "alias", ENGINE_CONN_MANAGER_SPRING_NAME ) - // TODO: group by key labels } @@ -81,12 +80,12 @@ class DefaultECMRegisterService extends ECMRegisterService with ECMEventListener } override def onEvent(event: ECMEvent): Unit = event match { - case event: ECMReadyEvent => registerECM(event) - case event: ECMClosedEvent => unRegisterECM(event) + case event: ECMReadyEvent => registerECM(readyEvent2RegisterECMRequest(event)) + case event: ECMClosedEvent => unRegisterECM(closeEvent2StopECMRequest(event)) case _ => } - private implicit def closeEvent2StopECMRequest(event: ECMClosedEvent): StopEMRequest = { + private def closeEvent2StopECMRequest(event: ECMClosedEvent): StopEMRequest = { val request = new StopEMRequest val instance = Sender.getThisServiceInstance request.setUser(Utils.getJvmUser) @@ -114,7 +113,10 @@ class DefaultECMRegisterService extends ECMRegisterService with ECMEventListener override def unRegisterECM(request: StopEMRequest): Unit = { logger.info("start unRegister ecm") - Sender.getSender(MANAGER_SERVICE_NAME).send(request) + if (!unRegisterFlag) { + Sender.getSender(MANAGER_SERVICE_NAME).send(request) + } + unRegisterFlag = true } } diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultEngineConnKillService.java b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultEngineConnKillService.java deleted file mode 100644 index 4f593736d9..0000000000 --- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultEngineConnKillService.java +++ /dev/null @@ -1,242 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.ecm.server.service.impl; - -import org.apache.commons.io.IOUtils; -import org.apache.linkis.common.ServiceInstance; -import org.apache.linkis.common.utils.Utils; -import org.apache.linkis.ecm.core.engineconn.EngineConn; -import org.apache.linkis.ecm.server.conf.ECMConfiguration; -import org.apache.linkis.ecm.server.service.EngineConnKillService; -import org.apache.linkis.ecm.server.service.EngineConnListService; -import org.apache.linkis.engineconn.common.conf.EngineConnConf; -import org.apache.linkis.governance.common.utils.GovernanceUtils; -import org.apache.linkis.manager.common.protocol.engine.EngineStopRequest; -import org.apache.linkis.manager.common.protocol.engine.EngineStopResponse; -import org.apache.linkis.manager.common.protocol.engine.EngineSuicideRequest; -import org.apache.linkis.manager.label.entity.Label; -import org.apache.linkis.manager.label.entity.engine.EngineTypeLabel; -import org.apache.linkis.rpc.message.annotation.Receiver; -import org.apache.linkis.rpc.Sender; -import org.apache.commons.lang3.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.BufferedReader; -import java.io.File; -import java.io.FileNotFoundException; -import java.io.FileReader; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -public class DefaultEngineConnKillService implements EngineConnKillService { - - private static final Logger logger = LoggerFactory.getLogger(DefaultEngineConnKillService.class); - - private EngineConnListService engineConnListService; - - public void setEngineConnListService(EngineConnListService engineConnListService) { - this.engineConnListService = engineConnListService; - } - - private static final ThreadPoolExecutor ecYarnAppKillService = Utils.newCachedThreadPool(10, "ECM-Kill-EC-Yarn-App", true); - - @Override - @Receiver - public EngineStopResponse dealEngineConnStop(EngineStopRequest engineStopRequest) { - logger.info("received EngineStopRequest " + engineStopRequest); - EngineConn engineConn = getEngineConnByServiceInstance(engineStopRequest.getServiceInstance()); - EngineStopResponse response = new EngineStopResponse(); - if (null != engineConn) { - if(!killEngineConnByPid(engineConn)) { - response.setStopStatus(false); - response.setMsg("Kill engine " + engineConn.getServiceInstance().toString() + " failed."); - } else { - response.setStopStatus(true); - response.setMsg("Kill engine " + engineConn.getServiceInstance().toString() + " succeed."); - } - killYarnAppIdOfOneEc(engineConn); - } else { - logger.warn("Cannot find engineconn : " + engineStopRequest.getServiceInstance().toString() + " in this engineConnManager engineConn list, cannot kill."); - response.setStopStatus(true); - response.setMsg("EngineConn " + engineStopRequest.getServiceInstance().toString() + " was not found in this engineConnManager."); - } - if (!response.getStopStatus()) { - EngineSuicideRequest request = new EngineSuicideRequest(engineStopRequest.getServiceInstance(), engineStopRequest.getUser()); - try { - Sender.getSender(engineStopRequest.getServiceInstance()).send(request); - response.setStopStatus(true); - response.setMsg(response.getMsg() + " Now send suicide request to engine."); - } catch (Exception e) { - response.setMsg(response.getMsg() + " Sended suicide request to engine error, " + e.getMessage()); - } - } - return response; - } - - public void killYarnAppIdOfOneEc(EngineConn engineConn) { - String engineConnInstance = engineConn.getServiceInstance().toString(); - logger.info("try to kill yarn app ids in the engine of ({}).", engineConnInstance); - String engineLogDir = engineConn.getEngineConnManagerEnv().engineConnLogDirs(); - final String errEngineLogPath = engineLogDir.concat(File.separator).concat("yarnApp.log"); - logger.info("try to parse the yarn app id from the engine err log file path: {}", errEngineLogPath); - File file = new File(errEngineLogPath); - if (file.exists()) - { - ecYarnAppKillService.execute(() -> { - BufferedReader in = null; - try { - in = new BufferedReader(new FileReader(errEngineLogPath)); - String line; - String regex = getYarnAppRegexByEngineType(engineConn); - if (StringUtils.isBlank(regex)) { - return; - } - Pattern pattern = Pattern.compile(regex); - List appIds = new ArrayList<>(); - while ((line = in.readLine()) != null) { - if (StringUtils.isNotBlank(line)) { - Matcher mApp = pattern.matcher(line); - if (mApp.find()) { - String candidate1 = mApp.group(mApp.groupCount()); - if (!appIds.contains(candidate1)) { - appIds.add(candidate1); - } - } - } - } - GovernanceUtils.killYarnJobApp(appIds); - logger.info("finished kill yarn app ids in the engine of ({}).", engineConnInstance); - } catch (IOException ioEx) { - if (ioEx instanceof FileNotFoundException) { - logger.error("the engine log file {} not found.", errEngineLogPath); - } else { - logger.error("the engine log file parse failed. the reason is {}", ioEx.getMessage()); - } - } finally { - IOUtils.closeQuietly(in); - } - }); - } - } - - private String getYarnAppRegexByEngineType(EngineConn engineConn) { - List> labels = engineConn.getLabels(); - String engineType = ""; - if (labels != null && !labels.isEmpty()) { - Optional labelOptional = labels.stream().filter(label -> label instanceof EngineTypeLabel) - .map(label -> (EngineTypeLabel) label).findFirst(); - if (labelOptional.isPresent()) { - EngineTypeLabel engineTypeLabel = labelOptional.get(); - engineType = engineTypeLabel.getEngineType(); - } - } - if (StringUtils.isBlank(engineType)) { - return ""; - } - String regex; - switch (engineType) { - case "spark": - case "shell": - regex = EngineConnConf.SPARK_ENGINE_CONN_YARN_APP_ID_PARSE_REGEX().getValue(); - break; - case "sqoop": - regex = EngineConnConf.SQOOP_ENGINE_CONN_YARN_APP_ID_PARSE_REGEX().getValue(); - break; - case "hive": - regex = EngineConnConf.HIVE_ENGINE_CONN_YARN_APP_ID_PARSE_REGEX().getValue(); - break; - default: - regex = ""; - } - return regex; - } - - private EngineConn getEngineConnByServiceInstance(ServiceInstance serviceInstance) { - if (null == serviceInstance) { - return null; - } - List engineConnList = engineConnListService.getEngineConns(); - for (EngineConn engineConn : engineConnList) { - if (null != engineConn && serviceInstance.equals(engineConn.getServiceInstance())) { - return engineConn; - } - } - return null; - } - - private boolean killEngineConnByPid(EngineConn engineConn) { - logger.info("try to kill {} toString with pid({}).", engineConn.getServiceInstance().toString(), engineConn.getPid()); - if (StringUtils.isNotBlank(engineConn.getPid())) { - if (ECMConfiguration.ECM_PROCESS_SCRIPT_KILL()) { - GovernanceUtils.killProcess(engineConn.getPid(), engineConn.getServiceInstance().toString(), true); - } else { - killProcessByKillCmd(engineConn.getPid(), engineConn.getServiceInstance().toString()); - } - if (isProcessAlive(engineConn.getPid())) { - return false; - } else { - return true; - } - } else { - logger.warn("cannot kill {} with empty pid.", engineConn.getServiceInstance().toString()); - return false; - } - } - - private boolean isProcessAlive(String pid) { - String findCmd = "ps -ef | grep " + pid + " | grep EngineConnServer | awk '{print \"exists_\"$2}' | grep " + pid; - List cmdList = new ArrayList<>(); - cmdList.add("bash"); - cmdList.add("-c"); - cmdList.add(findCmd); - try { - String rs = Utils.exec(cmdList.toArray(new String[0]), 5000L); - return null != rs && rs.contains("exists_" + pid); - } catch (Exception e) { - // todo when thread catch exception , it should not be return false - logger.warn("Method isProcessAlive failed, " + e.getMessage()); - return false; - } - } - - private void killProcessByKillCmd(String pid, String desc ) { - String k15cmd = "sudo kill " + pid; - String k9cmd = "sudo kill -9 " + pid; - int tryNum = 0; - try { - while (isProcessAlive(pid) && tryNum <= 3) { - logger.info("{} still alive with pid({}), use shell command to kill it. try {}++", desc, pid, tryNum++); - if (tryNum <= 3) { - Utils.exec(k15cmd.split(" "), 3000L); - } else { - logger.info("{} still alive with pid({}). try {}, use shell command to kill -9 it", desc, pid, tryNum); - Utils.exec(k9cmd.split(" "), 3000L); - } - Thread.sleep(5000); - } - } catch (InterruptedException e) { - logger.error("Interrupted while killing engine {} with pid({})." + desc, pid); - } - } -} diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultEngineConnListService.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultEngineConnListService.scala deleted file mode 100644 index 4b9a59b4d7..0000000000 --- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultEngineConnListService.scala +++ /dev/null @@ -1,192 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.ecm.server.service.impl - -import org.apache.linkis.DataWorkCloudApplication -import org.apache.linkis.common.utils.{Logging, Utils} -import org.apache.linkis.ecm.core.engineconn.{EngineConn, YarnEngineConn} -import org.apache.linkis.ecm.core.launch.EngineConnLaunchRunner -import org.apache.linkis.ecm.core.listener.{ECMEvent, ECMEventListener} -import org.apache.linkis.ecm.server.LinkisECMApplication -import org.apache.linkis.ecm.server.converter.ECMEngineConverter -import org.apache.linkis.ecm.server.listener._ -import org.apache.linkis.ecm.server.service.EngineConnListService -import org.apache.linkis.manager.common.entity.enumeration.NodeStatus -import org.apache.linkis.manager.common.entity.resource.{Resource, ResourceType} -import org.apache.linkis.manager.common.protocol.engine.EngineStopRequest - -import org.apache.commons.lang3.StringUtils - -import java.util -import java.util.concurrent.ConcurrentHashMap - -import scala.collection.JavaConverters._ - -import com.google.common.collect.Interners - -class DefaultEngineConnListService - extends EngineConnListService - with ECMEventListener - with Logging { - - /** - * key:tickedId,value :engineConn - */ - private val engineConnMap = new ConcurrentHashMap[String, EngineConn] - - private var engineConnKillService: DefaultEngineConnKillService = _ - - val lock = Interners.newWeakInterner[String] - - override def init(): Unit = {} - - override def getEngineConn(engineConnId: String): Option[EngineConn] = Option( - engineConnMap.get(engineConnId) - ) - - override def getEngineConns: util.List[EngineConn] = - new util.ArrayList[EngineConn](engineConnMap.values()) - - override def addEngineConn(engineConn: EngineConn): Unit = { - logger.info(s"add engineConn ${engineConn.getServiceInstance} to engineConnMap") - if (LinkisECMApplication.isReady) { - engineConnMap.put(engineConn.getTickedId, engineConn) - } - } - - override def killEngineConn(engineConnId: String): Unit = { - var conn = engineConnMap.get(engineConnId) - if (conn != null) engineConnId.intern().synchronized { - conn = engineConnMap.get(engineConnId) - if (conn != null) { - Utils.tryAndWarn { - if (NodeStatus.Failed == conn.getStatus && StringUtils.isNotBlank(conn.getPid)) { - killECByEngineConnKillService(conn) - } else { - getEngineConnKillService().killYarnAppIdOfOneEc(conn) - } - conn.close() - } - engineConnMap.remove(engineConnId) - logger.info(s"engineconn ${conn.getServiceInstance} was closed.") - } - } - } - - override def getUsedResources: Resource = engineConnMap - .values() - .asScala - .map(_.getResource.getMinResource) - .fold(Resource.initResource(ResourceType.Default))(_ + _) - - override def submit(runner: EngineConnLaunchRunner): Option[EngineConn] = { - None - } - - def updateYarnAppId(event: YarnAppIdCallbackEvent): Unit = { - updateYarnEngineConn( - x => x.setApplicationId(event.protocol.applicationId), - event.protocol.nodeId - ) - } - - def updateYarnEngineConn(implicit - updateFunction: YarnEngineConn => Unit, - nodeId: String - ): Unit = { - lock.intern(nodeId) synchronized { - engineConnMap.get(nodeId) match { - case e: YarnEngineConn => updateFunction(e) - case e: EngineConn => - engineConnMap.put(nodeId, ECMEngineConverter.engineConn2YarnEngineConn(e)) - } - } - } - - def updateEngineConn(updateFunction: EngineConn => Unit, nodeId: String): Unit = { - lock.intern(nodeId) synchronized { - engineConnMap.get(nodeId) match { - case e: EngineConn => updateFunction(e) - case _ => - } - } - } - - def updateYarnInfo(event: YarnInfoCallbackEvent): Unit = { - updateYarnEngineConn(x => x.setApplicationURL(event.protocol.uri), event.protocol.nodeId) - } - - def updatePid(event: EngineConnPidCallbackEvent): Unit = { - updateEngineConn( - x => { - x.setPid(event.protocol.pid) - x.setServiceInstance(event.protocol.serviceInstance) - }, - event.protocol.ticketId - ) - } - - def updateEngineConnStatus(tickedId: String, updateStatus: NodeStatus): Unit = { - updateEngineConn(x => x.setStatus(updateStatus), tickedId) - if (NodeStatus.isCompleted(updateStatus)) { - logger.info(s" from engineConnMap to remove engineconn ticketId ${tickedId}") - killEngineConn(tickedId) - } - } - - override def onEvent(event: ECMEvent): Unit = { - logger.info(s"Deal event $event") - event match { - case event: ECMClosedEvent => shutdownEngineConns(event) - case event: YarnAppIdCallbackEvent => updateYarnAppId(event) - case event: YarnInfoCallbackEvent => updateYarnInfo(event) - case event: EngineConnPidCallbackEvent => updatePid(event) - case EngineConnAddEvent(engineConn) => addEngineConn(engineConn) - case EngineConnStatusChangeEvent(tickedId, updateStatus) => - updateEngineConnStatus(tickedId, updateStatus) - case _ => - } - } - - private def getEngineConnKillService(): DefaultEngineConnKillService = { - if (engineConnKillService == null) { - val applicationContext = DataWorkCloudApplication.getApplicationContext - engineConnKillService = applicationContext.getBean(classOf[DefaultEngineConnKillService]) - } - engineConnKillService - } - - private def shutdownEngineConns(event: ECMClosedEvent): Unit = { - logger.info("start to kill all engines belonging the ecm") - engineConnMap - .values() - .asScala - .foreach(engineconn => { - killECByEngineConnKillService(engineconn) - }) - logger.info("Done! success to kill all engines belonging the ecm") - } - - private def killECByEngineConnKillService(engineconn: EngineConn): Unit = { - logger.info(s"start to kill ec by engineConnKillService ${engineconn.getServiceInstance}") - val engineStopRequest = new EngineStopRequest() - engineStopRequest.setServiceInstance(engineconn.getServiceInstance) - getEngineConnKillService().dealEngineConnStop(engineStopRequest) - } - -} diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultEngineConnPidCallbackService.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultEngineConnPidCallbackService.scala deleted file mode 100644 index 6fb2d4700d..0000000000 --- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultEngineConnPidCallbackService.scala +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.ecm.server.service.impl - -import org.apache.linkis.common.utils.Logging -import org.apache.linkis.ecm.server.LinkisECMApplication -import org.apache.linkis.ecm.server.listener.EngineConnPidCallbackEvent -import org.apache.linkis.ecm.server.service.EngineConnPidCallbackService -import org.apache.linkis.governance.common.protocol.task.ResponseEngineConnPid -import org.apache.linkis.rpc.message.annotation.Receiver - -class DefaultEngineConnPidCallbackService extends EngineConnPidCallbackService with Logging { - - @Receiver - override def dealPid(protocol: ResponseEngineConnPid): Unit = { - // 1.设置pid - // 2.设置serviceInstance - // 3.状态为running - LinkisECMApplication.getContext.getECMSyncListenerBus.postToAll( - EngineConnPidCallbackEvent(protocol) - ) - } - -} diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultEngineConnStatusCallbackService.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultEngineConnStatusCallbackService.scala deleted file mode 100644 index af627afece..0000000000 --- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultEngineConnStatusCallbackService.scala +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.ecm.server.service.impl - -import org.apache.linkis.common.utils.Logging -import org.apache.linkis.ecm.server.LinkisECMApplication -import org.apache.linkis.ecm.server.conf.ECMConfiguration.MANAGER_SERVICE_NAME -import org.apache.linkis.ecm.server.listener.EngineConnStatusChangeEvent -import org.apache.linkis.ecm.server.service.EngineConnStatusCallbackService -import org.apache.linkis.manager.common.entity.enumeration.NodeStatus -import org.apache.linkis.manager.common.entity.enumeration.NodeStatus.{Failed, Running} -import org.apache.linkis.manager.common.protocol.engine.{ - EngineConnStatusCallback, - EngineConnStatusCallbackToAM -} -import org.apache.linkis.rpc.Sender -import org.apache.linkis.rpc.message.annotation.Receiver - -import org.springframework.stereotype.Service - -@Service -class DefaultEngineConnStatusCallbackService extends EngineConnStatusCallbackService with Logging { - - @Receiver - override def dealEngineConnStatusCallback(protocol: EngineConnStatusCallback): Unit = { - logger.info(s"Start to deal EngineConnStatusCallback $protocol") - - if (NodeStatus.isAvailable(protocol.status)) { - - LinkisECMApplication.getContext.getECMSyncListenerBus.postToAll( - EngineConnStatusChangeEvent(protocol.ticketId, Running) - ) - } else { - - Sender - .getSender(MANAGER_SERVICE_NAME) - .send( - EngineConnStatusCallbackToAM( - protocol.serviceInstance, - protocol.status, - protocol.initErrorMsg - ) - ) - LinkisECMApplication.getContext.getECMSyncListenerBus.postToAll( - EngineConnStatusChangeEvent(protocol.ticketId, Failed) - ) - } - - logger.info(s"Finished to deal EngineConnStatusCallback $protocol") - } - -} diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultYarnCallbackService.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultYarnCallbackService.scala deleted file mode 100644 index 0bb2e1366f..0000000000 --- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultYarnCallbackService.scala +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.ecm.server.service.impl - -import org.apache.linkis.ecm.server.LinkisECMApplication -import org.apache.linkis.ecm.server.listener.{YarnAppIdCallbackEvent, YarnInfoCallbackEvent} -import org.apache.linkis.ecm.server.service.YarnCallbackService -import org.apache.linkis.protocol.callback.{YarnAPPIdCallbackProtocol, YarnInfoCallbackProtocol} -import org.apache.linkis.rpc.message.annotation.Receiver - -class DefaultYarnCallbackService extends YarnCallbackService { - - @Receiver - override def dealApplicationId(protocol: YarnAPPIdCallbackProtocol): Unit = { - LinkisECMApplication.getContext.getECMSyncListenerBus.postToAll( - YarnAppIdCallbackEvent(protocol) - ) - } - - @Receiver - override def dealApplicationURI(protocol: YarnInfoCallbackProtocol): Unit = { - LinkisECMApplication.getContext.getECMSyncListenerBus.postToAll(YarnInfoCallbackEvent(protocol)) - } - -} diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/ECMListenerService.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/ECMListenerService.scala new file mode 100644 index 0000000000..764a704887 --- /dev/null +++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/ECMListenerService.scala @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.ecm.server.service.impl + +import org.apache.linkis.common.utils.Logging +import org.apache.linkis.ecm.core.listener.{ECMEvent, ECMEventListener} +import org.apache.linkis.ecm.server.listener.EngineConnStopEvent +import org.apache.linkis.ecm.server.service.EngineConnKillService +import org.apache.linkis.manager.common.entity.enumeration.NodeStatus + +class ECMListenerService extends ECMEventListener with Logging { + + private var engineConnKillService: EngineConnKillService = _ + + override def onEvent(event: ECMEvent): Unit = event match { + case EngineConnStopEvent(engineConn, engineStopRequest) => + if (NodeStatus.Failed == engineConn.getStatus) { + logger.info("deal stopEvent to kill ec {}", engineStopRequest) + engineConnKillService.dealEngineConnStop(engineStopRequest) + } else { + if (engineConnKillService.isInstanceOf[DefaultEngineConnKillService]) { + logger.info("deal stopEvent to kill yarn app {}", engineStopRequest) + engineConnKillService + .asInstanceOf[DefaultEngineConnKillService] + .killYarnAppIdOfOneEc(engineStopRequest) + } + } + case _ => + } + + def getEngineConnKillService(): EngineConnKillService = { + engineConnKillService + } + + def setEngineConnKillService(engineConnKillService: EngineConnKillService): Unit = { + this.engineConnKillService = engineConnKillService + } + +} diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/ProcessEngineConnLaunchService.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/ProcessEngineConnLaunchService.scala index ad58ba723f..dbb65b0059 100644 --- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/ProcessEngineConnLaunchService.scala +++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/ProcessEngineConnLaunchService.scala @@ -19,31 +19,27 @@ package org.apache.linkis.ecm.server.service.impl import org.apache.linkis.common.conf.Configuration import org.apache.linkis.common.utils.Utils -import org.apache.linkis.ecm.core.conf.ECMErrorCode import org.apache.linkis.ecm.core.engineconn.EngineConn import org.apache.linkis.ecm.core.launch.ProcessEngineConnLaunch -import org.apache.linkis.ecm.errorcode.EngineconnServerErrorCodeSummary._ import org.apache.linkis.ecm.server.LinkisECMApplication import org.apache.linkis.ecm.server.conf.ECMConfiguration import org.apache.linkis.ecm.server.conf.ECMConfiguration.MANAGER_SERVICE_NAME -import org.apache.linkis.ecm.server.exception.ECMErrorException -import org.apache.linkis.ecm.server.listener.EngineConnStatusChangeEvent +import org.apache.linkis.ecm.server.listener.EngineConnStopEvent import org.apache.linkis.ecm.server.service.LocalDirsHandleService +import org.apache.linkis.governance.common.utils.{JobUtils, LoggerUtils} +import org.apache.linkis.manager.common.constant.AMConstant import org.apache.linkis.manager.common.entity.enumeration.NodeStatus -import org.apache.linkis.manager.common.entity.enumeration.NodeStatus._ -import org.apache.linkis.manager.common.protocol.engine.EngineConnStatusCallbackToAM +import org.apache.linkis.manager.common.protocol.engine.{ + EngineConnStatusCallbackToAM, + EngineStopRequest +} import org.apache.linkis.manager.engineplugin.common.launch.entity.EngineConnLaunchRequest import org.apache.linkis.manager.label.utils.LabelUtil import org.apache.linkis.rpc.Sender import org.apache.commons.io.IOUtils -import org.apache.commons.lang3.StringUtils -import org.apache.commons.lang3.exception.ExceptionUtils - -import java.util.concurrent.TimeUnit -import scala.concurrent.{Future, TimeoutException} -import scala.concurrent.duration.Duration +import scala.concurrent.Future abstract class ProcessEngineConnLaunchService extends AbstractEngineConnLaunchService { @@ -52,110 +48,78 @@ abstract class ProcessEngineConnLaunchService extends AbstractEngineConnLaunchSe def setLocalDirsHandleService(localDirsHandleService: LocalDirsHandleService): Unit = this.localDirsHandleService = localDirsHandleService - override def waitEngineConnStart( + override def startEngineConnMonitorStart( request: EngineConnLaunchRequest, - conn: EngineConn, - duration: Long + conn: EngineConn ): Unit = { conn.getEngineConnLaunchRunner.getEngineConnLaunch match { case launch: ProcessEngineConnLaunch => - Utils.tryCatch { - // Set the pid of the shell script before the pid callBack returns - launch.getPid().foreach(conn.setPid) - processMonitorThread(conn, launch, duration) - } { case e: Throwable => - val logPath = Utils.tryCatch(conn.getEngineConnManagerEnv.engineConnLogDirs) { t => - localDirsHandleService.getEngineConnLogDir( - request.user, - request.ticketId, - LabelUtil.getEngineType(request.labels) - ) - } - val canRetry = e match { - case ecmError: ECMErrorException => - if (ECMErrorCode.EC_START_TIME_OUT == ecmError.getErrCode) { - true - } else if (StringUtils.isBlank(ecmError.getDesc)) { - logger.info("exception desc is null, can be retry") - true - } else { - false - } - case _ => false - } + launch.getPid().foreach(conn.setPid) + processMonitorThread(conn, launch) + case _ => + } + } + + private def processMonitorThread( + engineConn: EngineConn, + launch: ProcessEngineConnLaunch + ): Unit = { + Future { + val tickedId = engineConn.getTickedId + val errorMsg = new StringBuilder + val taskId = + JobUtils.getJobIdFromStringMap(launch.getEngineConnLaunchRequest.creationDesc.properties) + LoggerUtils.setJobIdMDC(taskId) + Utils.tryAndWarnMsg { + val iterator = + IOUtils.lineIterator(launch.getProcessInputStream, Configuration.BDP_ENCODING.getValue) + var count = 0 + val maxLen = ECMConfiguration.ENGINE_START_ERROR_MSG_MAX_LEN.getValue + while (launch.isAlive && iterator.hasNext && count < maxLen) { + val line = iterator.next() + errorMsg.append(line).append("\n") + count += 1 + } + val exitCode = launch.processWaitFor + val engineType = LabelUtil.getEngineType(launch.getEngineConnLaunchRequest.labels) + val logPath = Utils.tryCatch(engineConn.getEngineConnManagerEnv.engineConnLogDirs) { t => + localDirsHandleService.getEngineConnLogDir( + launch.getEngineConnLaunchRequest.user, + tickedId, + engineType + ) + } + if (exitCode != 0) { + val canRetry = if (errorMsg.isEmpty) true else false logger.warn( - s"Failed to init ${conn.getServiceInstance}, status shutting down, canRetry $canRetry, logPath $logPath", - e + s"Failed to start ec ${engineConn.getServiceInstance}, status shutting down exit code ${exitCode}, canRetry ${canRetry}, logPath ${logPath}" ) Sender .getSender(MANAGER_SERVICE_NAME) .send( EngineConnStatusCallbackToAM( - conn.getServiceInstance, + engineConn.getServiceInstance, NodeStatus.ShuttingDown, - "Failed to start EngineConn, reason: " + ExceptionUtils.getRootCauseMessage( - e - ) + s"\n You can go to this path($logPath) to find the reason or ask the administrator for help", + "Failed to start EngineConn, reason: " + errorMsg + s"\n You can go to this path($logPath) to find the reason or ask the administrator for help", canRetry ) ) - throw e + engineConn.setStatus(NodeStatus.ShuttingDown) + } else { + engineConn.setStatus(NodeStatus.Success) } - case _ => - } - } - - private def processMonitorThread( - engineConn: EngineConn, - launch: ProcessEngineConnLaunch, - timeout: Long - ): Unit = { - val isCompleted: EngineConn => Boolean = engineConn => - engineConn.getStatus == Success || engineConn.getStatus == Failed - val tickedId = engineConn.getTickedId - val errorMsg = new StringBuilder - Future { - val iterator = - IOUtils.lineIterator(launch.getProcessInputStream, Configuration.BDP_ENCODING.getValue) - var count = 0 - val maxLen = ECMConfiguration.ENGINE_START_ERROR_MSG_MAX_LEN.getValue - while (!isCompleted(engineConn) && iterator.hasNext && count < maxLen) { - val line = iterator.next() - errorMsg.append(line).append("\n") - count += 1 - } - val exitCode = Option(launch.processWaitFor) - if (exitCode.exists(_ != 0)) { - logger.info(s"engine ${tickedId} process exit ") - LinkisECMApplication.getContext.getECMSyncListenerBus.postToAll( - EngineConnStatusChangeEvent(tickedId, ShuttingDown) - ) - } else { - LinkisECMApplication.getContext.getECMSyncListenerBus.postToAll( - EngineConnStatusChangeEvent(tickedId, Success) - ) - } - } - Utils.tryThrow( - Utils - .waitUntil(() => engineConn.getStatus != Starting, Duration(timeout, TimeUnit.MILLISECONDS)) - ) { - case e: TimeoutException => - throw new ECMErrorException( - EC_START_TIME_OUT.getErrorCode, - EC_START_TIME_OUT.getErrorDesc + s" $engineConn ." - ) - case e: InterruptedException => // 比如被ms cancel - throw new ECMErrorException( - EC_INTERRUPT_TIME_OUT.getErrorCode, - EC_INTERRUPT_TIME_OUT.getErrorDesc + s" $engineConn ." + val engineStopRequest = new EngineStopRequest + engineStopRequest.setEngineType(engineType) + engineStopRequest.setUser(launch.getEngineConnLaunchRequest.user) + engineStopRequest.setIdentifier(engineConn.getPid) + engineStopRequest.setIdentifierType(AMConstant.PROCESS_MARK) + engineStopRequest.setLogDirSuffix(logPath) + engineStopRequest.setServiceInstance(engineConn.getServiceInstance) + LinkisECMApplication.getContext.getECMAsyncListenerBus.post( + EngineConnStopEvent(engineConn, engineStopRequest) ) - case t: Throwable => - logger.error(s"unexpected error, now shutdown it.") - throw t - } - if (engineConn.getStatus == ShuttingDown) { - throw new ECMErrorException(EC_START_FAILED.getErrorCode, errorMsg.toString()) + } { s"EngineConns: ${engineConn.getServiceInstance} monitor Failed" } + LoggerUtils.removeJobIdMDC() } } diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/spring/ECMSpringConfiguration.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/spring/ECMSpringConfiguration.scala index 4a684bbec1..ec65cd885f 100644 --- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/spring/ECMSpringConfiguration.scala +++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/spring/ECMSpringConfiguration.scala @@ -19,7 +19,7 @@ package org.apache.linkis.ecm.server.spring import org.apache.linkis.ecm.core.listener.ECMEventListener import org.apache.linkis.ecm.server.context.{DefaultECMContext, ECMContext} -import org.apache.linkis.ecm.server.service._ +import org.apache.linkis.ecm.server.service.{EngineConnKillService, _} import org.apache.linkis.ecm.server.service.impl._ import org.springframework.beans.factory.annotation.Autowired @@ -39,16 +39,9 @@ class ECMSpringConfiguration { new DefaultECMContext } - @Bean - @ConditionalOnMissingBean - def getDefaultYarnCallbackService: YarnCallbackService = { - new DefaultYarnCallbackService - } - @Bean @ConditionalOnMissingBean def getBmlResourceLocalizationService( - context: ECMContext, localDirsHandleService: LocalDirsHandleService ): ResourceLocalizationService = { val service: BmlResourceLocalizationService = new BmlResourceLocalizationService @@ -57,32 +50,12 @@ class ECMSpringConfiguration { service } - @Bean - @ConditionalOnMissingBean - def getDefaultLogCallbackService: LogCallbackService = { - null - } - @Bean @ConditionalOnMissingBean def getDefaultlocalDirsHandleService: LocalDirsHandleService = { new DefaultLocalDirsHandleService } - @Bean - @ConditionalOnMissingBean - def getDefaultEngineConnPidCallbackService: EngineConnPidCallbackService = { - new DefaultEngineConnPidCallbackService - } - - @Bean - @ConditionalOnMissingBean - def getDefaultEngineConnListService(context: ECMContext): EngineConnListService = { - implicit val service: DefaultEngineConnListService = new DefaultEngineConnListService - registerSyncListener(context) - service - } - @Bean @ConditionalOnMissingBean def getLinuxProcessEngineConnLaunchService( @@ -98,38 +71,44 @@ class ECMSpringConfiguration { @Bean @ConditionalOnMissingBean def getDefaultECMRegisterService(context: ECMContext): ECMRegisterService = { - implicit val service: DefaultECMRegisterService = new DefaultECMRegisterService - registerSyncListener(context) + val service: DefaultECMRegisterService = new DefaultECMRegisterService + registerSyncListener(context, service) service } @Bean @ConditionalOnMissingBean def getDefaultECMHealthService(context: ECMContext): ECMHealthService = { - implicit val service: DefaultECMHealthService = new DefaultECMHealthService - registerSyncListener(context) + val service: DefaultECMHealthService = new DefaultECMHealthService + registerSyncListener(context, service) service } @Bean @ConditionalOnMissingBean def getDefaultEngineConnKillService( - engineConnListService: EngineConnListService ): EngineConnKillService = { val service = new DefaultEngineConnKillService - service.setEngineConnListService(engineConnListService) service } - private def registerSyncListener( + @Bean + @ConditionalOnMissingBean + def getECMListenerService( + engineConnKillService: EngineConnKillService, context: ECMContext - )(implicit listener: ECMEventListener): Unit = { + ): ECMListenerService = { + val service: ECMListenerService = new ECMListenerService + service.setEngineConnKillService(engineConnKillService) + registerASyncListener(context, service) + service + } + + private def registerSyncListener(context: ECMContext, listener: ECMEventListener): Unit = { context.getECMSyncListenerBus.addListener(listener) } - private def registerASyncListener( - context: ECMContext - )(implicit listener: ECMEventListener): Unit = { + private def registerASyncListener(context: ECMContext, listener: ECMEventListener): Unit = { context.getECMAsyncListenerBus.addListener(listener) } diff --git a/linkis-computation-governance/linkis-engineconn/linkis-clustered-engineconn/linkis-once-engineconn/src/main/scala/org/apache/linkis/engineconn/once/executor/hook/OnceEngineConnHook.scala b/linkis-computation-governance/linkis-engineconn/linkis-clustered-engineconn/linkis-once-engineconn/src/main/scala/org/apache/linkis/engineconn/once/executor/hook/OnceEngineConnHook.scala index cd4029bae6..2586576bb7 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-clustered-engineconn/linkis-once-engineconn/src/main/scala/org/apache/linkis/engineconn/once/executor/hook/OnceEngineConnHook.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-clustered-engineconn/linkis-once-engineconn/src/main/scala/org/apache/linkis/engineconn/once/executor/hook/OnceEngineConnHook.scala @@ -28,11 +28,6 @@ import org.apache.linkis.manager.common.entity.enumeration.NodeStatus */ class OnceEngineConnHook extends CallbackEngineConnHook { - override protected def getNodeStatusOfStartSuccess( - engineCreationContext: EngineCreationContext, - engineConn: EngineConn - ): NodeStatus = NodeStatus.Unlock - override def afterEngineServerStartSuccess( engineCreationContext: EngineCreationContext, engineConn: EngineConn diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/java/org/apache/linkis/engineconn/computation/concurrent/monitor/TimingMonitorService.java b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/java/org/apache/linkis/engineconn/computation/concurrent/monitor/TimingMonitorService.java index de6bb440dd..979b19ed46 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/java/org/apache/linkis/engineconn/computation/concurrent/monitor/TimingMonitorService.java +++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/java/org/apache/linkis/engineconn/computation/concurrent/monitor/TimingMonitorService.java @@ -55,7 +55,7 @@ public class TimingMonitorService implements InitializingBean, Runnable { @Override public void afterPropertiesSet() throws Exception { - if (AccessibleExecutorConfiguration.ENGINECONN_SUPPORT_PARALLELISM()) { + if ((Boolean) (AccessibleExecutorConfiguration.ENGINECONN_SUPPORT_PARALLELISM().getValue())) { Utils.defaultScheduler() .scheduleAtFixedRate( this, 3 * 60 * 1000, MONITOR_INTERVAL.getValue().toLong(), TimeUnit.MILLISECONDS); diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/async/AsyncEngineConnJob.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/async/AsyncEngineConnJob.scala index 8876a50c37..6f73f67fe6 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/async/AsyncEngineConnJob.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/async/AsyncEngineConnJob.scala @@ -20,6 +20,7 @@ package org.apache.linkis.engineconn.computation.executor.async import org.apache.linkis.common.utils.Utils import org.apache.linkis.engineconn.computation.executor.entity.EngineConnTask import org.apache.linkis.engineconn.computation.executor.execute.EngineExecutionContext +import org.apache.linkis.governance.common.utils.{JobUtils, LoggerUtils} import org.apache.linkis.scheduler.executer.{ CompletedExecuteResponse, ErrorExecuteResponse, @@ -27,6 +28,7 @@ import org.apache.linkis.scheduler.executer.{ SuccessExecuteResponse } import org.apache.linkis.scheduler.queue.{Job, JobInfo} +import org.apache.linkis.scheduler.queue.SchedulerEventState.SchedulerEventState class AsyncEngineConnJob(task: EngineConnTask, engineExecutionContext: EngineExecutionContext) extends Job { @@ -47,6 +49,14 @@ class AsyncEngineConnJob(task: EngineConnTask, engineExecutionContext: EngineExe override def close(): Unit = {} + override def transition(state: SchedulerEventState): Unit = Utils.tryFinally { + val jobId = JobUtils.getJobIdFromMap(task.getProperties) + LoggerUtils.setJobIdMDC(jobId) + super.transition(state) + } { + LoggerUtils.removeJobIdMDC() + } + override def transitionCompleted(executeCompleted: CompletedExecuteResponse): Unit = { var executeCompletedNew: CompletedExecuteResponse = executeCompleted executeCompleted match { diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/async/AsyncExecuteExecutor.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/async/AsyncExecuteExecutor.scala index a27d3f029a..46332b93fd 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/async/AsyncExecuteExecutor.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/async/AsyncExecuteExecutor.scala @@ -17,8 +17,10 @@ package org.apache.linkis.engineconn.computation.executor.async +import org.apache.linkis.common.utils.Utils import org.apache.linkis.engineconn.common.exception.EngineConnException import org.apache.linkis.engineconn.computation.executor.utlis.ComputationErrorCode +import org.apache.linkis.governance.common.utils.{JobUtils, LoggerUtils} import org.apache.linkis.scheduler.executer._ import org.apache.linkis.scheduler.executer.ExecutorState.ExecutorState @@ -31,10 +33,16 @@ class AsyncExecuteExecutor(executor: AsyncConcurrentComputationExecutor) extends override def execute(executeRequest: ExecuteRequest): ExecuteResponse = { executeRequest match { case asyncExecuteRequest: AsyncExecuteRequest => - executor.asyncExecuteTask( - asyncExecuteRequest.task, - asyncExecuteRequest.engineExecutionContext - ) + Utils.tryFinally { + val jobId = JobUtils.getJobIdFromMap(asyncExecuteRequest.task.getProperties) + LoggerUtils.setJobIdMDC(jobId) + executor.asyncExecuteTask( + asyncExecuteRequest.task, + asyncExecuteRequest.engineExecutionContext + ) + } { + LoggerUtils.removeJobIdMDC() + } case _ => throw EngineConnException( ComputationErrorCode.ASYNC_EXECUTOR_ERROR_CODE, diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/bml/BmlEnginePreExecuteHook.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/bml/BmlEnginePreExecuteHook.scala index 3959eb942b..98f04daaa2 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/bml/BmlEnginePreExecuteHook.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/bml/BmlEnginePreExecuteHook.scala @@ -20,6 +20,7 @@ package org.apache.linkis.engineconn.computation.executor.bml import org.apache.linkis.bml.client.{BmlClient, BmlClientFactory} import org.apache.linkis.common.exception.ErrorException import org.apache.linkis.common.utils.{Logging, Utils} +import org.apache.linkis.engineconn.common.conf.EngineConnConf import org.apache.linkis.engineconn.common.creation.EngineCreationContext import org.apache.linkis.engineconn.computation.executor.execute.EngineExecutionContext import org.apache.linkis.engineconn.computation.executor.hook.ComputationExecutorHook @@ -62,7 +63,7 @@ class BmlEnginePreExecuteHook extends ComputationExecutorHook with Logging { ): String = { val props = engineExecutionContext.getProperties if (null != props && props.containsKey(GovernanceConstant.TASK_RESOURCES_STR)) { - val workDir = ComputationEngineUtils.getCurrentWorkDir +// val workDir = ComputationEngineUtils.getCurrentWorkDir val jobId = engineExecutionContext.getJobId props.get(GovernanceConstant.TASK_RESOURCES_STR) match { case resources: util.List[Object] => @@ -71,9 +72,9 @@ class BmlEnginePreExecuteHook extends ComputationExecutorHook with Logging { val fileName = resource.get(GovernanceConstant.TASK_RESOURCE_FILE_NAME_STR).toString val resourceId = resource.get(GovernanceConstant.TASK_RESOURCE_ID_STR).toString val version = resource.get(GovernanceConstant.TASK_RESOURCE_VERSION_STR).toString - val fullPath = - if (workDir.endsWith(seperator)) pathType + workDir + fileName - else pathType + workDir + seperator + fileName + val fullPath = fileName +// if (workDir.endsWith(seperator)) pathType + workDir + fileName +// else pathType + workDir + seperator + fileName val response = Utils.tryCatch { bmlClient.downloadShareResource(processUser, resourceId, version, fullPath, true) } { diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/cs/CSResourceParser.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/cs/CSResourceParser.scala index f59adaadef..fe98e3328e 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/cs/CSResourceParser.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/cs/CSResourceParser.scala @@ -17,7 +17,10 @@ package org.apache.linkis.engineconn.computation.executor.cs +import org.apache.linkis.common.utils.Logging import org.apache.linkis.cs.client.service.CSResourceService +import org.apache.linkis.engineconn.common.conf.EngineConnConf +import org.apache.linkis.governance.common.utils.GovernanceConstant import org.apache.commons.lang3.StringUtils @@ -27,7 +30,7 @@ import java.util.regex.Pattern import scala.collection.JavaConverters._ import scala.collection.mutable.ArrayBuffer -class CSResourceParser { +class CSResourceParser extends Logging { private val pb = Pattern.compile("cs://[^\\s\"]+[$\\s]{0,1}", Pattern.CASE_INSENSITIVE) @@ -47,7 +50,6 @@ class CSResourceParser { nodeNameStr: String ): String = { - // TODO getBMLResource val bmlResourceList = CSResourceService.getInstance().getUpstreamBMLResource(contextIDValueStr, nodeNameStr) @@ -56,23 +58,25 @@ class CSResourceParser { val preFixNames = new ArrayBuffer[String]() val parsedNames = new ArrayBuffer[String]() + val prefixName = System.currentTimeMillis().toString + "_" preFixResourceNames.foreach { preFixResourceName => val resourceName = preFixResourceName.replace(PREFIX, "").trim val bmlResourceOption = bmlResourceList.asScala.find(_.getDownloadedFileName.equals(resourceName)) if (bmlResourceOption.isDefined) { + val replacementName = EngineConnConf.getEngineTmpDir + prefixName + resourceName val bmlResource = bmlResourceOption.get val map = new util.HashMap[String, Object]() - map.put("resourceId", bmlResource.getResourceId) - map.put("version", bmlResource.getVersion) - map.put("fileName", resourceName) + map.put(GovernanceConstant.TASK_RESOURCE_ID_STR, bmlResource.getResourceId) + map.put(GovernanceConstant.TASK_RESOURCE_VERSION_STR, bmlResource.getVersion) + map.put(GovernanceConstant.TASK_RESOURCE_FILE_NAME_STR, replacementName) parsedResources.add(map) preFixNames.append(preFixResourceName) - parsedNames.append(resourceName) + parsedNames.append(replacementName) + logger.warn(s"Replace cs file from {$preFixResourceName} to {$replacementName}") } - } - props.put("resources", parsedResources) + props.put(GovernanceConstant.TASK_RESOURCES_STR, parsedResources) StringUtils.replaceEach(code, preFixNames.toArray, parsedNames.toArray) } diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/execute/ComputationExecutor.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/execute/ComputationExecutor.scala index 780db4215a..96e5a7d480 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/execute/ComputationExecutor.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/execute/ComputationExecutor.scala @@ -21,7 +21,11 @@ import org.apache.linkis.DataWorkCloudApplication import org.apache.linkis.common.log.LogUtils import org.apache.linkis.common.utils.{Logging, Utils} import org.apache.linkis.engineconn.acessible.executor.entity.AccessibleExecutor -import org.apache.linkis.engineconn.acessible.executor.listener.event.TaskStatusChangedEvent +import org.apache.linkis.engineconn.acessible.executor.listener.event.{ + TaskLogUpdateEvent, + TaskResponseErrorEvent, + TaskStatusChangedEvent +} import org.apache.linkis.engineconn.common.conf.{EngineConnConf, EngineConnConstant} import org.apache.linkis.engineconn.computation.executor.conf.ComputationExecutorConf import org.apache.linkis.engineconn.computation.executor.entity.EngineConnTask @@ -35,6 +39,7 @@ import org.apache.linkis.engineconn.executor.listener.ExecutorListenerBusContext import org.apache.linkis.governance.common.entity.ExecutionNodeStatus import org.apache.linkis.governance.common.paser.CodeParser import org.apache.linkis.governance.common.protocol.task.{EngineConcurrentInfo, RequestTask} +import org.apache.linkis.governance.common.utils.{JobUtils, LoggerUtils} import org.apache.linkis.manager.common.entity.enumeration.NodeStatus import org.apache.linkis.manager.label.entity.engine.UserCreatorLabel import org.apache.linkis.protocol.engine.JobProgressInfo @@ -131,6 +136,12 @@ abstract class ComputationExecutor(val outputPrintLimit: Int = 1000) override def close(): Unit = { if (null != lastTask) CLOSE_LOCKER.synchronized { + listenerBusContext.getEngineConnSyncListenerBus.postToAll( + TaskLogUpdateEvent( + lastTask.getTaskId, + LogUtils.generateERROR("EC exits unexpectedly and actively kills the task") + ) + ) killTask(lastTask.getTaskId) } else { @@ -242,12 +253,15 @@ abstract class ComputationExecutor(val outputPrintLimit: Int = 1000) response = response match { case _: OutputExecuteResponse => succeedTasks.increase() - transformTaskStatus(engineConnTask, ExecutionNodeStatus.Succeed) SuccessExecuteResponse() case s: SuccessExecuteResponse => succeedTasks.increase() - transformTaskStatus(engineConnTask, ExecutionNodeStatus.Succeed) s + case incompleteExecuteResponse: IncompleteExecuteResponse => + ErrorExecuteResponse( + s"The task cannot be an incomplete response ${incompleteExecuteResponse.message}", + null + ) case _ => response } response @@ -257,20 +271,34 @@ abstract class ComputationExecutor(val outputPrintLimit: Int = 1000) } } - def execute(engineConnTask: EngineConnTask): ExecuteResponse = { + def execute(engineConnTask: EngineConnTask): ExecuteResponse = Utils.tryFinally { + val jobId = JobUtils.getJobIdFromMap(engineConnTask.getProperties) + LoggerUtils.setJobIdMDC(jobId) logger.info(s"start to execute task ${engineConnTask.getTaskId}") updateLastActivityTime() beforeExecute(engineConnTask) taskCache.put(engineConnTask.getTaskId, engineConnTask) lastTask = engineConnTask val response = ensureOp { - toExecuteTask(engineConnTask) + val executeResponse = toExecuteTask(engineConnTask) + executeResponse match { + case successExecuteResponse: SuccessExecuteResponse => + transformTaskStatus(engineConnTask, ExecutionNodeStatus.Succeed) + case errorExecuteResponse: ErrorExecuteResponse => + listenerBusContext.getEngineConnSyncListenerBus.postToAll( + TaskResponseErrorEvent(engineConnTask.getTaskId, errorExecuteResponse.message) + ) + transformTaskStatus(engineConnTask, ExecutionNodeStatus.Failed) + } + executeResponse } Utils.tryAndWarn(afterExecute(engineConnTask, response)) logger.info(s"Finished to execute task ${engineConnTask.getTaskId}") // lastTask = null response + } { + LoggerUtils.removeJobIdMDC() } def setCodeParser(codeParser: CodeParser): Unit = this.codeParser = Some(codeParser) diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/execute/EngineExecutionContext.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/execute/EngineExecutionContext.scala index 08124f2225..b97bb1cd9e 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/execute/EngineExecutionContext.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/execute/EngineExecutionContext.scala @@ -188,7 +188,6 @@ class EngineExecutionContext(executor: ComputationExecutor, executorUser: String logger.info(log) } else { val listenerBus = getEngineSyncListenerBus - // jobId.foreach(jId => listenerBus.post(TaskLogUpdateEvent(jId, log))) getJobId.foreach(jId => listenerBus.postToAll(TaskLogUpdateEvent(jId, log))) } diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/ComputationEngineConnHook.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/ComputationEngineConnHook.scala index d22bf3f800..e5ccd2bfbe 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/ComputationEngineConnHook.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/ComputationEngineConnHook.scala @@ -25,11 +25,6 @@ import org.apache.linkis.manager.common.entity.enumeration.NodeStatus class ComputationEngineConnHook extends CallbackEngineConnHook { - override protected def getNodeStatusOfStartSuccess( - engineCreationContext: EngineCreationContext, - engineConn: EngineConn - ): NodeStatus = NodeStatus.Unlock - override def afterEngineServerStartSuccess( engineCreationContext: EngineCreationContext, engineConn: EngineConn diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/metrics/ComputationEngineConnMetrics.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/metrics/ComputationEngineConnMetrics.scala index f96896f557..4446bdc672 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/metrics/ComputationEngineConnMetrics.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/metrics/ComputationEngineConnMetrics.scala @@ -91,4 +91,12 @@ object ComputationEngineConnMetrics { getTotalBusyTimeMills(nodeStatus) + getTotalIdleTimeMills(nodeStatus) def getUnlockToShutdownDurationMills(): Long = unlockToShutdownDurationMills.get() + + def getLastUnlockTimestamp(nodeStatus: NodeStatus): Long = { + nodeStatus match { + case NodeStatus.Unlock => lastUnlockTimeMills + case _ => 0 + } + } + } diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/service/DefaultNodeHeartbeatMsgManager.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/service/DefaultNodeHeartbeatMsgManager.scala index eccf54bfad..e5d74282de 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/service/DefaultNodeHeartbeatMsgManager.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/service/DefaultNodeHeartbeatMsgManager.scala @@ -21,10 +21,13 @@ import org.apache.linkis.common.utils.{Logging, Utils} import org.apache.linkis.engineconn.acessible.executor.info.NodeHeartbeatMsgManager import org.apache.linkis.engineconn.computation.executor.metrics.ComputationEngineConnMetrics import org.apache.linkis.engineconn.core.EngineConnObject -import org.apache.linkis.engineconn.executor.entity.{Executor, SensibleExecutor} +import org.apache.linkis.engineconn.executor.entity.{Executor, SensibleExecutor, YarnExecutor} import org.apache.linkis.governance.common.constant.ec.ECConstants +import org.apache.linkis.manager.common.entity.enumeration.NodeStatus import org.apache.linkis.server.BDPJettyServerHelper +import org.apache.commons.lang3.StringUtils + import org.springframework.stereotype.Component import java.util @@ -63,6 +66,10 @@ class DefaultNodeHeartbeatMsgManager extends NodeHeartbeatMsgManager with Loggin ECConstants.EC_TOTAL_LOCK_TIME_MILLS_KEY, ComputationEngineConnMetrics.getTotalLockTimeMills(status).asInstanceOf[Object] ) + msgMap.put( + ECConstants.EC_LAST_UNLOCK_TIMESTAMP, + ComputationEngineConnMetrics.getLastUnlockTimestamp(status).asInstanceOf[Object] + ) case _ => } val engineParams = EngineConnObject.getEngineCreationContext.getOptions @@ -72,6 +79,22 @@ class DefaultNodeHeartbeatMsgManager extends NodeHeartbeatMsgManager with Loggin engineParams.get(ECConstants.YARN_QUEUE_NAME_CONFIG_KEY).asInstanceOf[Object] ) } + executor match { + case yarnExecutor: YarnExecutor => + if (StringUtils.isNotBlank(yarnExecutor.getQueue)) { + msgMap.put(ECConstants.YARN_QUEUE_NAME_KEY, yarnExecutor.getQueue) + } + if (StringUtils.isNotBlank(yarnExecutor.getApplicationId)) { + msgMap.put(ECConstants.YARN_APPID_NAME_KEY, yarnExecutor.getApplicationId) + } + if (StringUtils.isNotBlank(yarnExecutor.getApplicationURL)) { + msgMap.put(ECConstants.YARN_APP_URL_KEY, yarnExecutor.getApplicationURL) + } + if (StringUtils.isNotBlank(yarnExecutor.getYarnMode)) { + msgMap.put(ECConstants.YARN_MODE_KEY, yarnExecutor.getYarnMode) + } + case _ => + } Utils.tryCatch(BDPJettyServerHelper.gson.toJson(msgMap)) { case e: Exception => val msgs = msgMap.asScala .map { case (k, v) => if (null == v) s"${k}->null" else s"${k}->${v.toString}" } diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/service/TaskExecutionServiceImpl.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/service/TaskExecutionServiceImpl.scala index 651fc0f3dc..00ee5e28e5 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/service/TaskExecutionServiceImpl.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/service/TaskExecutionServiceImpl.scala @@ -58,7 +58,7 @@ import org.apache.linkis.governance.common.exception.engineconn.{ EngineConnExecutorErrorException } import org.apache.linkis.governance.common.protocol.task._ -import org.apache.linkis.governance.common.utils.JobUtils +import org.apache.linkis.governance.common.utils.{JobUtils, LoggerUtils} import org.apache.linkis.manager.common.entity.enumeration.NodeStatus import org.apache.linkis.manager.common.protocol.resource.{ ResponseTaskRunningInfo, @@ -170,82 +170,92 @@ class TaskExecutionServiceImpl } @Receiver - override def execute(requestTask: RequestTask, sender: Sender): ExecuteResponse = { - - // check lock - logger.info("Received a new task, task content is " + requestTask) - if (StringUtils.isBlank(requestTask.getLock)) { - logger.error(s"Invalid lock : ${requestTask.getLock} , requestTask : " + requestTask) - return ErrorExecuteResponse( - s"Invalid lock : ${requestTask.getLock}.", - new EngineConnExecutorErrorException( - EngineConnExecutorErrorCode.INVALID_PARAMS, - "Invalid lock or code(请获取到锁后再提交任务.)" + override def execute(requestTask: RequestTask, sender: Sender): ExecuteResponse = + Utils.tryFinally { + val jobId = JobUtils.getJobIdFromMap(requestTask.getProperties) + LoggerUtils.setJobIdMDC(jobId) + // check lock + logger.info("Received a new task, task content is " + requestTask) + if (StringUtils.isBlank(requestTask.getLock)) { + logger.error(s"Invalid lock : ${requestTask.getLock} , requestTask : " + requestTask) + return ErrorExecuteResponse( + s"Invalid lock : ${requestTask.getLock}.", + new EngineConnExecutorErrorException( + EngineConnExecutorErrorCode.INVALID_PARAMS, + "Invalid lock or code(请获取到锁后再提交任务.)" + ) ) - ) - } - if (!lockService.isLockExist(requestTask.getLock)) { - logger.error(s"Lock ${requestTask.getLock} not exist, cannot execute.") - return ErrorExecuteResponse( - "Lock not exixt", - new EngineConnExecutorErrorException( - EngineConnExecutorErrorCode.INVALID_LOCK, - "Lock : " + requestTask.getLock + " not exist(您的锁无效，请重新获取后再提交)." + } + if (!lockService.isLockExist(requestTask.getLock)) { + logger.error(s"Lock ${requestTask.getLock} not exist, cannot execute.") + return ErrorExecuteResponse( + "Lock not exixt", + new EngineConnExecutorErrorException( + EngineConnExecutorErrorCode.INVALID_LOCK, + "Lock : " + requestTask.getLock + " not exist(您的锁无效，请重新获取后再提交)." + ) ) - ) - } + } - if (StringUtils.isBlank(requestTask.getCode)) { - return IncompleteExecuteResponse( - "Your code is incomplete, it may be that only comments are selected for execution(您的代码不完整，可能是仅仅选中了注释进行执行)" - ) - } + if (StringUtils.isBlank(requestTask.getCode)) { + return IncompleteExecuteResponse( + "Your code is incomplete, it may be that only comments are selected for execution(您的代码不完整，可能是仅仅选中了注释进行执行)" + ) + } - val taskId: Int = taskExecutedNum.incrementAndGet() - val retryAble: Boolean = { - val retry = - requestTask.getProperties.getOrDefault(ComputationEngineConstant.RETRYABLE_TYPE_NAME, null) - if (null != retry) retry.asInstanceOf[Boolean] - else false - } - val jobId = JobUtils.getJobIdFromMap(requestTask.getProperties) - if (StringUtils.isNotBlank(jobId)) { - System.getProperties.put(ComputationExecutorConf.JOB_ID_TO_ENV_KEY, jobId) - logger.info(s"Received job with id ${jobId}.") - } - val task = new CommonEngineConnTask(String.valueOf(taskId), retryAble) - task.setCode(requestTask.getCode) - task.setProperties(requestTask.getProperties) - task.data(ComputationEngineConstant.LOCK_TYPE_NAME, requestTask.getLock) - task.setStatus(ExecutionNodeStatus.Scheduled) - val labels = requestTask.getLabels.asScala.toArray - task.setLabels(labels) - val entranceServerInstance = RPCUtils.getServiceInstanceFromSender(sender) - task.setCallbackServiceInstance(entranceServerInstance) - logger.info(s"task $taskId submit executor to execute") - val runnable = new Runnable { - override def run(): Unit = Utils.tryCatch { - // Waiting to run, preventing task messages from being sent to submit services before SubmitResponse, such as entry - Thread.sleep(ComputationExecutorConf.TASK_SUBMIT_WAIT_TIME_MS) - submitTaskToExecutor(task, labels) match { - case ErrorExecuteResponse(message, throwable) => - sendToEntrance(task, ResponseTaskError(task.getTaskId, message)) - logger.error(message, throwable) - sendToEntrance(task, ResponseTaskStatus(task.getTaskId, ExecutionNodeStatus.Failed)) - case _ => + val taskId: Int = taskExecutedNum.incrementAndGet() + val retryAble: Boolean = { + val retry = + requestTask.getProperties.getOrDefault( + ComputationEngineConstant.RETRYABLE_TYPE_NAME, + null + ) + if (null != retry) retry.asInstanceOf[Boolean] + else false + } + + if (StringUtils.isNotBlank(jobId)) { + System.getProperties.put(ComputationExecutorConf.JOB_ID_TO_ENV_KEY, jobId) + logger.info(s"Received job with id ${jobId}.") + } + val task = new CommonEngineConnTask(String.valueOf(taskId), retryAble) + task.setCode(requestTask.getCode) + task.setProperties(requestTask.getProperties) + task.data(ComputationEngineConstant.LOCK_TYPE_NAME, requestTask.getLock) + task.setStatus(ExecutionNodeStatus.Scheduled) + val labels = requestTask.getLabels.asScala.toArray + task.setLabels(labels) + val entranceServerInstance = RPCUtils.getServiceInstanceFromSender(sender) + task.setCallbackServiceInstance(entranceServerInstance) + logger.info(s"task $taskId submit executor to execute") + val runnable = new Runnable { + override def run(): Unit = Utils.tryCatch { + // Waiting to run, preventing task messages from being sent to submit services before SubmitResponse, such as entry + Thread.sleep(ComputationExecutorConf.TASK_SUBMIT_WAIT_TIME_MS) + LoggerUtils.setJobIdMDC(jobId) + submitTaskToExecutor(task, labels) match { + case ErrorExecuteResponse(message, throwable) => + sendToEntrance(task, ResponseTaskError(task.getTaskId, message)) + logger.error(message, throwable) + sendToEntrance(task, ResponseTaskStatus(task.getTaskId, ExecutionNodeStatus.Failed)) + case _ => + } + LoggerUtils.removeJobIdMDC() + } { t => + logger.warn("Failed to submit task ", t) + LoggerUtils.removeJobIdMDC() + sendToEntrance( + task, + ResponseTaskError(task.getTaskId, ExceptionUtils.getRootCauseMessage(t)) + ) + sendToEntrance(task, ResponseTaskStatus(task.getTaskId, ExecutionNodeStatus.Failed)) } - } { t => - logger.warn("Failed to submit task ", t) - sendToEntrance( - task, - ResponseTaskError(task.getTaskId, ExceptionUtils.getRootCauseMessage(t)) - ) - sendToEntrance(task, ResponseTaskStatus(task.getTaskId, ExecutionNodeStatus.Failed)) } + val submitTaskToExecutorFuture = taskAsyncSubmitExecutor.submit(runnable) + SubmitResponse(task.getTaskId) + } { + LoggerUtils.removeJobIdMDC() } - val submitTaskToExecutorFuture = taskAsyncSubmitExecutor.submit(runnable) - SubmitResponse(task.getTaskId) - } private def submitTaskToExecutor( task: CommonEngineConnTask, @@ -256,11 +266,11 @@ class TaskExecutionServiceImpl case computationExecutor: ComputationExecutor => taskIdCache.put(task.getTaskId, computationExecutor) submitTask(task, computationExecutor) - case o => + case _ => val labelsStr = if (labels != null) labels.filter(_ != null).map(_.getStringValue).mkString(",") else "" val msg = - "Invalid computationExecutor : " + o.getClass.getName + ", labels : " + labelsStr + ", requestTask : " + task.getTaskId + "Invalid computationExecutor : " + executor.getClass.getName + ", labels : " + labelsStr + ", requestTask : " + task.getTaskId logger.error(msg) ErrorExecuteResponse( "Invalid computationExecutor(生成无效的计算引擎，请联系管理员).", @@ -388,18 +398,15 @@ class TaskExecutionServiceImpl new Thread(consumerRunnable) } - private def executeTask(task: EngineConnTask, executor: ComputationExecutor): Unit = { - val response = executor.execute(task) - response match { - case ErrorExecuteResponse(message, throwable) => - sendToEntrance(task, ResponseTaskError(task.getTaskId, message)) - logger.error(message, throwable) - LogHelper.pushAllRemainLogs() - executor.transformTaskStatus(task, ExecutionNodeStatus.Failed) - case _ => logger.warn(s"task get response is $response") + private def executeTask(task: EngineConnTask, executor: ComputationExecutor): Unit = + Utils.tryFinally { + val jobId = JobUtils.getJobIdFromMap(task.getProperties) + LoggerUtils.setJobIdMDC(jobId) + executor.execute(task) + clearCache(task.getTaskId) + } { + LoggerUtils.removeJobIdMDC() } - clearCache(task.getTaskId) - } /** * Open daemon thread @@ -417,72 +424,91 @@ class TaskExecutionServiceImpl ): Future[_] = { val sleepInterval = ComputationExecutorConf.ENGINE_PROGRESS_FETCH_INTERVAL.getValue scheduler.submit(new Runnable { - override def run(): Unit = Utils.tryAndWarn { + override def run(): Unit = { Utils.tryQuietly(Thread.sleep(TimeUnit.MILLISECONDS.convert(1, TimeUnit.SECONDS))) while (null != taskFuture && !taskFuture.isDone) { - if ( - ExecutionNodeStatus.isCompleted(task.getStatus) || ExecutionNodeStatus - .isRunning(task.getStatus) - ) { - val progressResponse = taskProgress(task.getTaskId) - val resourceResponse: ResponseTaskYarnResource = - taskYarnResource(task.getTaskId) match { - case responseTaskYarnResource: ResponseTaskYarnResource => - if ( - responseTaskYarnResource.resourceMap != null && !responseTaskYarnResource.resourceMap.isEmpty - ) { - responseTaskYarnResource - } else { - null - } - case _ => + if (!ExecutionNodeStatus.isCompleted(task.getStatus)) { + Utils.tryAndWarn { + val progressResponse = Utils.tryCatch(taskProgress(task.getTaskId)) { + case e: Exception => + logger.info("Failed to get progress", e) null } - val extraInfoMap = new util.HashMap[String, Object]() - extraInfoMap.put(TaskConstant.ENGINE_INSTANCE, Sender.getThisInstance) - extraInfoMap.put( - ECConstants.EC_TICKET_ID_KEY, - EngineConnObject.getEngineCreationContext.getTicketId - ) - val ecParams = EngineConnObject.getEngineCreationContext.getOptions - if (ecParams.containsKey(ECConstants.YARN_QUEUE_NAME_CONFIG_KEY)) { - extraInfoMap.put( - ECConstants.YARN_QUEUE_NAME_KEY, - ecParams.get(ECConstants.YARN_QUEUE_NAME_CONFIG_KEY) - ) - } - extraInfoMap.put(TaskConstant.ENGINE_CONN_TASK_ID, task.getTaskId) - extraInfoMap.put( - TaskConstant.ENGINE_CONN_SUBMIT_TIME, - System.currentTimeMillis.toString - ) - // todo add other info - var respRunningInfo: ResponseTaskRunningInfo = null - if (null != resourceResponse) { - respRunningInfo = ResponseTaskRunningInfo( - progressResponse.execId, - progressResponse.progress, - progressResponse.progressInfo, - resourceResponse.resourceMap, - extraInfoMap - ) - } else { - respRunningInfo = ResponseTaskRunningInfo( - progressResponse.execId, - progressResponse.progress, - progressResponse.progressInfo, - null, + val resourceResponse = Utils.tryCatch(buildResourceMap(task)) { case e: Exception => + logger.info("Failed to get resource", e) + null + } + val extraInfoMap = Utils.tryCatch(buildExtraInfoMap(task)) { case e: Exception => + logger.info("Failed to get extra info ", e) + null + } + val resourceMap = if (null != resourceResponse) resourceResponse.resourceMap else null + + /** + * It is guaranteed that there must be progress the progress must be greater than or + * equal to 0.1 + */ + val newProgressResponse = if (null == progressResponse) { + ResponseTaskProgress(task.getTaskId, 0.1f, null) + } else if (progressResponse.progress < 0.1f) { + ResponseTaskProgress(task.getTaskId, 0.1f, progressResponse.progressInfo) + } else { + progressResponse + } + val respRunningInfo: ResponseTaskRunningInfo = ResponseTaskRunningInfo( + newProgressResponse.execId, + newProgressResponse.progress, + newProgressResponse.progressInfo, + resourceMap, extraInfoMap ) + sendToEntrance(task, respRunningInfo) } - sendToEntrance(task, respRunningInfo) - Thread.sleep(TimeUnit.MILLISECONDS.convert(sleepInterval, TimeUnit.SECONDS)) } + Utils.tryQuietly( + Thread.sleep(TimeUnit.MILLISECONDS.convert(sleepInterval, TimeUnit.SECONDS)) + ) } } }) } + private def buildExtraInfoMap(task: EngineConnTask): util.HashMap[String, Object] = { + val extraInfoMap = new util.HashMap[String, Object]() + extraInfoMap.put(TaskConstant.ENGINE_INSTANCE, Sender.getThisInstance) + extraInfoMap.put( + ECConstants.EC_TICKET_ID_KEY, + EngineConnObject.getEngineCreationContext.getTicketId + ) + val ecParams = EngineConnObject.getEngineCreationContext.getOptions + if (ecParams.containsKey(ECConstants.YARN_QUEUE_NAME_CONFIG_KEY)) { + extraInfoMap.put( + ECConstants.YARN_QUEUE_NAME_KEY, + ecParams.get(ECConstants.YARN_QUEUE_NAME_CONFIG_KEY) + ) + } + extraInfoMap.put(TaskConstant.ENGINE_CONN_TASK_ID, task.getTaskId) + extraInfoMap.put(TaskConstant.ENGINE_CONN_SUBMIT_TIME, System.currentTimeMillis.toString) + extraInfoMap + } + + private def buildResourceMap(task: EngineConnTask): ResponseTaskYarnResource = { + val resourceResponse: ResponseTaskYarnResource = + taskYarnResource(task.getTaskId) match { + case responseTaskYarnResource: ResponseTaskYarnResource => + if ( + responseTaskYarnResource.resourceMap != null && !responseTaskYarnResource.resourceMap.isEmpty + ) { + responseTaskYarnResource + } else { + null + } + case _ => + null + } + resourceResponse + } + private def taskYarnResource(taskID: String): ResponseTaskYarnResource = { val executor = taskIdCache.getIfPresent(taskID) executor match { @@ -494,7 +520,7 @@ class TaskExecutionServiceImpl } override def taskProgress(taskID: String): ResponseTaskProgress = { - var response = ResponseTaskProgress(taskID, 0, null) + var response = ResponseTaskProgress(taskID, 0.01f, null) if (StringUtils.isBlank(taskID)) return response val executor = taskIdCache.getIfPresent(taskID) if (null != executor) { @@ -503,19 +529,15 @@ class TaskExecutionServiceImpl if (ExecutionNodeStatus.isCompleted(task.getStatus)) { response = ResponseTaskProgress(taskID, 1.0f, null) } else { + val progress = executor.progress(taskID) + logger.info("The latest progress {} of the task id {} is:", progress, taskID) response = Utils.tryQuietly( - ResponseTaskProgress( - taskID, - executor.progress(taskID), - executor.getProgressInfo(taskID) - ) + ResponseTaskProgress(taskID, progress, executor.getProgressInfo(taskID)) ) } - } else { - response = ResponseTaskProgress(taskID, -1, null) } } else { - logger.error(s"Executor of taskId : $taskID is not cached.") + logger.info(s"Executor of taskId : $taskID is not cached.") } response } @@ -603,7 +625,7 @@ class TaskExecutionServiceImpl logger.warn("Unknown event : " + BDPJettyServerHelper.gson.toJson(event)) } - override def onLogUpdate(logUpdateEvent: TaskLogUpdateEvent): Unit = { + override def onLogUpdate(logUpdateEvent: TaskLogUpdateEvent): Unit = Utils.tryAndWarn { if (EngineConnConf.ENGINE_PUSH_LOG_TO_ENTRANCE.getValue) { if (null != logUpdateEvent && StringUtils.isNotBlank(logUpdateEvent.taskId)) { val task = getTaskByTaskId(logUpdateEvent.taskId) @@ -660,26 +682,33 @@ class TaskExecutionServiceImpl } } - override def onProgressUpdate(taskProgressUpdateEvent: TaskProgressUpdateEvent): Unit = { - if (EngineConnConf.ENGINE_PUSH_LOG_TO_ENTRANCE.getValue) { - val task = getTaskByTaskId(taskProgressUpdateEvent.taskId) - if (null != task) { - sendToEntrance( - task, - ResponseTaskProgress( + override def onProgressUpdate(taskProgressUpdateEvent: TaskProgressUpdateEvent): Unit = + Utils.tryAndWarn { + if (EngineConnConf.ENGINE_PUSH_LOG_TO_ENTRANCE.getValue) { + val task = getTaskByTaskId(taskProgressUpdateEvent.taskId) + if (null != task) { + val resourceResponse = buildResourceMap(task) + val extraInfoMap = buildExtraInfoMap(task) + + val resourceMap = if (null != resourceResponse) resourceResponse.resourceMap else null + + val respRunningInfo: ResponseTaskRunningInfo = ResponseTaskRunningInfo( taskProgressUpdateEvent.taskId, taskProgressUpdateEvent.progress, - taskProgressUpdateEvent.progressInfo + taskProgressUpdateEvent.progressInfo, + resourceMap, + extraInfoMap ) - ) - } else { - logger.error( - "Task cannot null! taskProgressUpdateEvent : " + ComputationEngineUtils.GSON - .toJson(taskProgressUpdateEvent) - ) + + sendToEntrance(task, respRunningInfo) + } else { + logger.error( + "Task cannot null! taskProgressUpdateEvent : " + ComputationEngineUtils.GSON + .toJson(taskProgressUpdateEvent) + ) + } } } - } override def onResultSetCreated(taskResultCreateEvent: TaskResultCreateEvent): Unit = { logger.info(s"start to deal result event ${taskResultCreateEvent.taskId}") diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/upstream/handler/ECTaskKillHandler.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/upstream/handler/ECTaskKillHandler.scala index d06e8ac077..15e70315e3 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/upstream/handler/ECTaskKillHandler.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/upstream/handler/ECTaskKillHandler.scala @@ -39,10 +39,10 @@ class ECTaskKillHandler extends MonitorHandler with Logging { while (elements.hasNext) { val element = elements.next Utils.tryCatch { - doKill(element) logger.error( s"ERROR: entrance : ${element.getUpstreamConnection().getUpstreamServiceInstanceName()} lose connect, will kill job : ${element.getKey()}" ) + doKill(element) } { t => logger.error("Failed to kill job: " + element.getKey, t) } diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/test/scala/org/apache/linkis/engineconn/computation/executor/upstream/access/ECTaskEntranceInfoAccessHelper.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/test/scala/org/apache/linkis/engineconn/computation/executor/upstream/access/ECTaskEntranceInfoAccessHelper.scala deleted file mode 100644 index 42e79c52cc..0000000000 --- a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/test/scala/org/apache/linkis/engineconn/computation/executor/upstream/access/ECTaskEntranceInfoAccessHelper.scala +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.engineconn.computation.executor.upstream.access - -import org.apache.linkis.DataWorkCloudApplication -import org.apache.linkis.common.ServiceInstance -import org.apache.linkis.common.conf.{CommonVars, DWCArgumentsParser} -import org.apache.linkis.common.utils.Utils -import org.apache.linkis.engineconn.common.creation.DefaultEngineCreationContext -import org.apache.linkis.engineconn.core.util.EngineConnUtils -import org.apache.linkis.governance.common.conf.GovernanceCommonConf -import org.apache.linkis.governance.common.utils.EngineConnArgumentsParser -import org.apache.linkis.manager.engineplugin.common.launch.process.Environment -import org.apache.linkis.manager.label.builder.factory.{ - LabelBuilderFactory, - LabelBuilderFactoryContext -} -import org.apache.linkis.manager.label.entity.Label -import org.apache.linkis.server.conf.ServerConfiguration - -import org.apache.commons.lang3.StringUtils - -import java.util - -import org.slf4j.{Logger, LoggerFactory} - -object ECTaskEntranceInfoAccessHelper { - val logger: Logger = LoggerFactory.getLogger(ECTaskEntranceInfoAccessHelper.getClass) - - val engineCreationContext = new DefaultEngineCreationContext - val labelBuilderFactory: LabelBuilderFactory = LabelBuilderFactoryContext.getLabelBuilderFactory - - def initApp(args: Array[String]): Unit = { - val arguments = EngineConnArgumentsParser.getEngineConnArgumentsParser.parseToObj(args) - val engineConf = arguments.getEngineConnConfMap - engineCreationContext.setUser(engineConf.getOrElse("user", Utils.getJvmUser)) - engineCreationContext.setTicketId(engineConf.getOrElse("ticketId", "")) - val host = CommonVars(Environment.ECM_HOST.toString, "127.0.0.1").getValue - val port = CommonVars(Environment.ECM_PORT.toString, "80").getValue - engineCreationContext.setEMInstance( - ServiceInstance(GovernanceCommonConf.ENGINE_CONN_MANAGER_SPRING_NAME.getValue, s"$host:$port") - ) - val labels = new util.ArrayList[Label[_]] - val labelArgs = engineConf.filter(_._1.startsWith(EngineConnArgumentsParser.LABEL_PREFIX)) - if (labelArgs.nonEmpty) { - labelArgs.foreach { case (key, value) => - labels.add( - labelBuilderFactory - .createLabel[Label[_]](key.replace(EngineConnArgumentsParser.LABEL_PREFIX, ""), value) - ) - } - engineCreationContext.setLabels(labels) - } - val jMap = new java.util.HashMap[String, String](engineConf.size) - engineConf.foreach(kv => jMap.put(kv._1, kv._2)) - engineCreationContext.setOptions(jMap) - engineCreationContext.setArgs(args) - // EngineConnObject.setEngineCreationContext(engineCreationContext) - logger.info( - "Finished to init engineCreationContext: " + EngineConnUtils.GSON - .toJson(engineCreationContext) - ) - - logger.info("Spring is enabled, now try to start SpringBoot.") - logger.info("<--------------------Start SpringBoot App-------------------->") - val parser = DWCArgumentsParser.parse(engineCreationContext.getArgs) - DWCArgumentsParser.setDWCOptionMap(parser.getDWCConfMap) - val existsExcludePackages = ServerConfiguration.BDP_SERVER_EXCLUDE_PACKAGES.getValue - if (!StringUtils.isEmpty(existsExcludePackages)) { - DataWorkCloudApplication.setProperty( - ServerConfiguration.BDP_SERVER_EXCLUDE_PACKAGES.key, - existsExcludePackages - ) - } - // 加载spring类 - DataWorkCloudApplication.main(DWCArgumentsParser.formatSpringOptions(parser.getSpringConfMap)) - - logger.info("<--------------------SpringBoot App init succeed-------------------->") - } - -} diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-common/src/main/scala/org/apache/linkis/engineconn/common/conf/EngineConnConf.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-common/src/main/scala/org/apache/linkis/engineconn/common/conf/EngineConnConf.scala index 1518f00a6c..b134fa5638 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-common/src/main/scala/org/apache/linkis/engineconn/common/conf/EngineConnConf.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-common/src/main/scala/org/apache/linkis/engineconn/common/conf/EngineConnConf.scala @@ -37,7 +37,7 @@ object EngineConnConf { val ENGINE_CONN_ONCE_HOOKS = CommonVars( "linkis.engine.connector.once.hooks", - "org.apache.linkis.engineconn.computation.executor.hook.ComputationEngineConnHook" + "org.apache.linkis.engineconn.once.executor.hook.OnceEngineConnHook" ) val ENGINE_LAUNCH_CMD_PARAMS_USER_KEY = @@ -61,6 +61,8 @@ object EngineConnConf { val ENGINE_CONN_LOCAL_LOG_DIRS_KEY = CommonVars("wds.linkis.engine.logs.dir.key", "LOG_DIRS") + val ENGINE_CONN_LOCAL_TMP_DIR = CommonVars("wds.linkis.engine.tmp.dir", "TEMP_DIRS") + val ENGINE_CONN_CREATION_WAIT_TIME = CommonVars("wds.linkis.engine.connector.init.time", new TimeType("8m")) @@ -81,6 +83,8 @@ object EngineConnConf { def getWorkHome: String = System.getenv(ENGINE_CONN_LOCAL_PATH_PWD_KEY.getValue) + def getEngineTmpDir: String = System.getenv(ENGINE_CONN_LOCAL_TMP_DIR.getValue) + def getLogDir: String = { val logDir = System.getenv(ENGINE_CONN_LOCAL_LOG_DIRS_KEY.getValue) if (StringUtils.isNotEmpty(logDir)) logDir else new File(getWorkHome, "logs").getPath diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-core/src/main/scala/org/apache/linkis/engineconn/core/executor/ExecutorManager.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-core/src/main/scala/org/apache/linkis/engineconn/core/executor/ExecutorManager.scala index f758b129e3..61242beaae 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-core/src/main/scala/org/apache/linkis/engineconn/core/executor/ExecutorManager.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-core/src/main/scala/org/apache/linkis/engineconn/core/executor/ExecutorManager.scala @@ -137,7 +137,7 @@ class LabelExecutorManagerImpl extends LabelExecutorManager with Logging { } protected def getLabelKey(labels: Array[Label[_]]): String = - labels.map(_.getStringValue).mkString("&") + labels.filter(null != _).map(_.getStringValue).mkString("&") protected def createExecutor( engineCreationContext: EngineCreationContext, @@ -171,7 +171,10 @@ class LabelExecutorManagerImpl extends LabelExecutorManager with Logging { MessageFormat.format(CANNOT_GET_LABEL_KEY.getErrorDesc, GSON.toJson(labels)) ) } - + if (!executors.isEmpty && factories.size <= 1) { + logger.info("For a single Executor EC, if an Executor exists, it will be returned directly") + return getReportExecutor.asInstanceOf[LabelExecutor] + } if (!executors.containsKey(labelKey)) executors synchronized { if (!executors.containsKey(labelKey)) { val executor = tryCreateExecutor(engineCreationContext, labels) @@ -184,6 +187,12 @@ class LabelExecutorManagerImpl extends LabelExecutorManager with Logging { override def generateExecutorId(): Int = idCreator.getAndIncrement() override def getExecutorByLabels(labels: Array[Label[_]]): LabelExecutor = { + + if (!executors.isEmpty && factories.size <= 1) { + logger.info("For a single Executor EC, if an Executor exists, it will be returned directly") + return getReportExecutor.asInstanceOf[LabelExecutor] + } + val labelKey = getLabelKey(labels) if (null == labelKey) return null if (!executors.containsKey(labelKey)) executors synchronized { diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-core/src/main/scala/org/apache/linkis/engineconn/launch/EngineConnServer.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-core/src/main/scala/org/apache/linkis/engineconn/launch/EngineConnServer.scala index 180798a772..14a0701d9d 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-core/src/main/scala/org/apache/linkis/engineconn/launch/EngineConnServer.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-core/src/main/scala/org/apache/linkis/engineconn/launch/EngineConnServer.scala @@ -127,6 +127,7 @@ object EngineConnServer extends Logging { this.engineCreationContext.setEMInstance( ServiceInstance(GovernanceCommonConf.ENGINE_CONN_MANAGER_SPRING_NAME.getValue, s"$host:$port") ) + val labels = new ArrayBuffer[Label[_]] val labelArgs = engineConf.filter(_._1.startsWith(EngineConnArgumentsParser.LABEL_PREFIX)) if (labelArgs.nonEmpty) { diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/conf/AccessibleExecutorConfiguration.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/conf/AccessibleExecutorConfiguration.scala index 0eb211f731..2d3a614434 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/conf/AccessibleExecutorConfiguration.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/conf/AccessibleExecutorConfiguration.scala @@ -37,13 +37,13 @@ object AccessibleExecutorConfiguration { val ENGINECONN_LOG_SEND_SIZE = CommonVars[Int]("wds.linkis.engineconn.log.send.cache.size", 300) val ENGINECONN_MAX_FREE_TIME = - CommonVars("wds.linkis.engineconn.max.free.time", new TimeType("30m")) + CommonVars("wds.linkis.engineconn.max.free.time", new TimeType("5m")) val ENGINECONN_LOCK_CHECK_INTERVAL = CommonVars("wds.linkis.engineconn.lock.free.interval", new TimeType("3m")) - val ENGINECONN_SUPPORT_PARALLELISM: Boolean = - CommonVars("wds.linkis.engineconn.support.parallelism", false).getValue + val ENGINECONN_SUPPORT_PARALLELISM = + CommonVars("wds.linkis.engineconn.support.parallelism", false) val ENGINECONN_HEARTBEAT_TIME = CommonVars("wds.linkis.engineconn.heartbeat.time", new TimeType("2m")) @@ -55,4 +55,16 @@ object AccessibleExecutorConfiguration { val ENABLE_MAINTAIN_CREATORS = CommonVars("wds.linkis.engineconn.maintain.cretors", "IDE") + val REPORTING_DELAY_MS = CommonVars( + "linkis.engineconn.heartbeat.report.delay", + 20, + "Heartbeat status reporting delay, default 20ms, Negative numbers do not take effect" + ).getValue + + val REPORTING_IGNORE_MS = CommonVars( + "linkis.engineconn.heartbeat.report.ignore", + 3, + "Heartbeat status report repeated ignore, default 3ms,Negative numbers do not take effect" + ).getValue + } diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/conf/AccessibleExecutorSpringConfiguration.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/conf/AccessibleExecutorSpringConfiguration.scala index 53cdd44b05..93cb41f344 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/conf/AccessibleExecutorSpringConfiguration.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/conf/AccessibleExecutorSpringConfiguration.scala @@ -43,9 +43,13 @@ class AccessibleExecutorSpringConfiguration extends Logging { def createLockManager(): LockService = { val lockService = - if (AccessibleExecutorConfiguration.ENGINECONN_SUPPORT_PARALLELISM) { + if (AccessibleExecutorConfiguration.ENGINECONN_SUPPORT_PARALLELISM.getHotValue()) { + logger.info("Engine supports parallelism.") new EngineConnConcurrentLockService - } else new EngineConnTimedLockService + } else { + logger.info("Engine doesn't support parallelism.") + new EngineConnTimedLockService + } asyncListenerBusContext.addListener(lockService) lockService } diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/hook/OperationHook.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/hook/OperationHook.scala new file mode 100644 index 0000000000..12e42c66a5 --- /dev/null +++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/hook/OperationHook.scala @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineconn.acessible.executor.hook + +import org.apache.linkis.manager.common.protocol.engine.{ + EngineOperateRequest, + EngineOperateResponse +} + +import scala.collection.mutable.ArrayBuffer + +trait OperationHook { + def getName(): String + + def doPreOperation( + engineOperateRequest: EngineOperateRequest, + engineOperateResponse: EngineOperateResponse + ): Unit + + def doPostOperation( + engineOperateRequest: EngineOperateRequest, + engineOperateResponse: EngineOperateResponse + ): Unit + +} + +object OperationHook { + private var operationHooks: ArrayBuffer[OperationHook] = new ArrayBuffer[OperationHook]() + + def registerOperationHook(operationHook: OperationHook): Unit = { + operationHooks.append(operationHook) + } + + def getOperationHooks(): Array[OperationHook] = operationHooks.toArray +} diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/lock/EngineConnTimedLock.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/lock/EngineConnTimedLock.scala index 26a9203795..af4d1eb017 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/lock/EngineConnTimedLock.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/lock/EngineConnTimedLock.scala @@ -28,6 +28,7 @@ import org.apache.linkis.engineconn.acessible.executor.listener.event.{ ExecutorUnLockEvent } import org.apache.linkis.engineconn.core.executor.ExecutorManager +import org.apache.linkis.engineconn.executor.entity.SensibleExecutor import org.apache.linkis.engineconn.executor.listener.ExecutorListenerBusContext import org.apache.linkis.manager.common.entity.enumeration.NodeStatus @@ -42,12 +43,10 @@ class EngineConnTimedLock(private var timeout: Long) val releaseScheduler = new ScheduledThreadPoolExecutor(1) var releaseTask: ScheduledFuture[_] = null var lastLockTime: Long = 0 - var lockedBy: AccessibleExecutor = null override def acquire(executor: AccessibleExecutor): Unit = { lock.acquire() lastLockTime = System.currentTimeMillis() - lockedBy = executor scheduleTimeout } @@ -57,8 +56,6 @@ class EngineConnTimedLock(private var timeout: Long) logger.debug("try to lock for succeed is " + succeed.toString) if (succeed) { lastLockTime = System.currentTimeMillis() - lockedBy = executor - logger.debug("try to lock for add time out task ! Locked by thread : " + lockedBy.getId) scheduleTimeout } succeed @@ -67,18 +64,13 @@ class EngineConnTimedLock(private var timeout: Long) // Unlock callback is not called in release method, because release method is called actively override def release(): Unit = { logger.debug( - "try to release for lock," + lockedBy + ",current thread " + Thread.currentThread().getName + s"try to release for lock: ${lock.toString}, current thread " + Thread.currentThread().getName ) - if (lockedBy != null) { - // && lockedBy == Thread.currentThread() Inconsistent thread(线程不一致) - logger.debug("try to release for lockedBy and thread ") - if (releaseTask != null) { - releaseTask.cancel(true) - releaseTask = null - } - logger.debug("try to release for lock release success") - lockedBy = null + if (releaseTask != null) { + releaseTask.cancel(true) + releaseTask = null } + logger.debug("try to release for lock release success") unlockCallback(lock.toString) resetLock() } @@ -96,7 +88,6 @@ class EngineConnTimedLock(private var timeout: Long) releaseScheduler.purge() } lock.release() - lockedBy = null } resetLock() } @@ -108,13 +99,18 @@ class EngineConnTimedLock(private var timeout: Long) new Runnable { override def run(): Unit = { synchronized { - if (isAcquired() && NodeStatus.Idle == lockedBy.getStatus && isExpired()) { - // unlockCallback depends on lockedBy, so lockedBy cannot be set null before unlockCallback - logger.info(s"Lock : [${lock.toString} was released due to timeout.") - release() - } else if (isAcquired() && NodeStatus.Busy == lockedBy.getStatus) { - lastLockTime = System.currentTimeMillis() - logger.info("Update lastLockTime because executor is busy.") + ExecutorManager.getInstance.getReportExecutor match { + case reportExecutor: AccessibleExecutor => + if ( + isAcquired() && NodeStatus.Idle == reportExecutor.getStatus && isExpired() + ) { + // unlockCallback depends on lockedBy, so lockedBy cannot be set null before unlockCallback + logger.info(s"Lock : [${lock.toString} was released due to timeout.") + release() + } else if (isAcquired() && NodeStatus.Busy == reportExecutor.getStatus) { + lastLockTime = System.currentTimeMillis() + logger.info("Update lastLockTime because executor is busy.") + } } } } @@ -143,14 +139,12 @@ class EngineConnTimedLock(private var timeout: Long) } override def renew(): Boolean = { - if (lockedBy != null) { - if (isAcquired && releaseTask != null) { - if (releaseTask.cancel(false)) { - releaseScheduler.purge() - scheduleTimeout - lastLockTime = System.currentTimeMillis() - return true - } + if (isAcquired && releaseTask != null) { + if (releaseTask.cancel(false)) { + releaseScheduler.purge() + scheduleTimeout + lastLockTime = System.currentTimeMillis() + return true } } false @@ -169,9 +163,18 @@ class EngineConnTimedLock(private var timeout: Long) } private def unlockCallback(lockStr: String): Unit = { - /* if (null != lockedBy) { - lockedBy.transition(NodeStatus.Unlock) - } */ + val nodeStatus = ExecutorManager.getInstance.getReportExecutor match { + case sensibleExecutor: SensibleExecutor => + sensibleExecutor.getStatus + case _ => NodeStatus.Idle + } + if (NodeStatus.isCompleted(nodeStatus)) { + logger.info( + "The node({}) is already in the completed state, and the unlocking is invalid", + nodeStatus.toString + ) + return + } val executors = ExecutorManager.getInstance.getExecutors.filter(executor => null != executor && !executor.isClosed ) @@ -185,7 +188,7 @@ class EngineConnTimedLock(private var timeout: Long) ExecutorListenerBusContext .getExecutorListenerBusContext() .getEngineConnAsyncListenerBus - .post(ExecutorUnLockEvent(null, lockStr.toString)) + .post(ExecutorUnLockEvent(null, lockStr)) } override def onExecutorCreated(executorCreateEvent: ExecutorCreateEvent): Unit = {} diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/operator/impl/EngineConnApplicationInfoOperator.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/operator/impl/EngineConnApplicationInfoOperator.scala index ad762892ef..c7635615e0 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/operator/impl/EngineConnApplicationInfoOperator.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/operator/impl/EngineConnApplicationInfoOperator.scala @@ -17,9 +17,11 @@ package org.apache.linkis.engineconn.acessible.executor.operator.impl +import org.apache.linkis.engineconn.acessible.executor.service.OperateService import org.apache.linkis.engineconn.common.exception.EngineConnException import org.apache.linkis.engineconn.core.executor.ExecutorManager import org.apache.linkis.engineconn.executor.entity.YarnExecutor +import org.apache.linkis.governance.common.constant.ec.ECConstants._ import org.apache.linkis.manager.common.operator.Operator class EngineConnApplicationInfoOperator extends Operator { @@ -30,10 +32,10 @@ class EngineConnApplicationInfoOperator extends Operator { ExecutorManager.getInstance.getReportExecutor match { case yarnExecutor: YarnExecutor => Map( - "applicationId" -> yarnExecutor.getApplicationId, - "applicationUrl" -> yarnExecutor.getApplicationURL, - "queue" -> yarnExecutor.getQueue, - "yarnMode" -> yarnExecutor.getYarnMode + YARN_APPID_NAME_KEY -> yarnExecutor.getApplicationId, + YARN_APP_URL_KEY -> yarnExecutor.getApplicationURL, + QUEUE -> yarnExecutor.getQueue, + YARN_MODE_KEY -> yarnExecutor.getYarnMode ) case _ => throw EngineConnException( diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/DefaultAccessibleService.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/DefaultAccessibleService.scala index d07d16ce27..8ef944fc9c 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/DefaultAccessibleService.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/DefaultAccessibleService.scala @@ -19,6 +19,7 @@ package org.apache.linkis.engineconn.acessible.executor.service import org.apache.linkis.DataWorkCloudApplication import org.apache.linkis.common.utils.{Logging, Utils} +import org.apache.linkis.engineconn.acessible.executor.conf.AccessibleExecutorConfiguration import org.apache.linkis.engineconn.acessible.executor.entity.AccessibleExecutor import org.apache.linkis.engineconn.acessible.executor.listener.event.{ ExecutorCompletedEvent, @@ -57,6 +58,12 @@ class DefaultAccessibleService extends AccessibleService with Logging { private var shutDownHooked: Boolean = false + private var lastStatusChanged: Long = System.currentTimeMillis() + + private var lastStatus: NodeStatus = null + + private var lastThreadName: String = null + @Receiver override def dealEngineStopRequest( engineSuicideRequest: EngineSuicideRequest, @@ -67,7 +74,7 @@ class DefaultAccessibleService extends AccessibleService with Logging { DataWorkCloudApplication.getServiceInstance.equals(engineSuicideRequest.getServiceInstance) ) { stopEngine() - logger.info(s"engine will suiside now.") + logger.info(s"engine was asked to suiside by ${engineSuicideRequest.getUser} now.") ShutdownHook.getShutdownHook.notifyStop() } else { if (null != engineSuicideRequest.getServiceInstance) { @@ -167,7 +174,39 @@ class DefaultAccessibleService extends AccessibleService with Logging { override def onExecutorStatusChanged( executorStatusChangedEvent: ExecutorStatusChangedEvent ): Unit = { - reportHeartBeatMsg(executorStatusChangedEvent.executor) + val sinceLastTime = System.currentTimeMillis() - lastStatusChanged + val reportDelay = AccessibleExecutorConfiguration.REPORTING_DELAY_MS + if ( + reportDelay > 0 && executorStatusChangedEvent.toStatus != lastStatus && reportDelay > sinceLastTime + ) { + logger.info( + "In order to ensure that the previous state is consumed first, sleep here {} ms", + reportDelay * 2 + ) + + Thread.sleep(reportDelay * 2) + } + val ignoreTime = AccessibleExecutorConfiguration.REPORTING_IGNORE_MS + val currentThreadName = Thread.currentThread().getName + if ( + ignoreTime > 0 && executorStatusChangedEvent.toStatus == lastStatus && ignoreTime > sinceLastTime && currentThreadName + .equals(lastThreadName) + ) { + logger.info( + "If the status is the same and the time is short and the thread is the same, no status report is performed {}", + executorStatusChangedEvent + ) + } else if ( + NodeStatus.Busy == lastStatus && executorStatusChangedEvent.toStatus == NodeStatus.Idle + ) { + logger.info("The state transition from Busy to Idle is not reported") + } else { + reportHeartBeatMsg(executorStatusChangedEvent.executor) + } + logger.info("Finished to report status {}", executorStatusChangedEvent) + lastStatusChanged = System.currentTimeMillis() + lastStatus = executorStatusChangedEvent.toStatus + lastThreadName = currentThreadName } private def reportHeartBeatMsg(executor: Executor): Unit = { diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/DefaultExecutorHeartbeatService.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/DefaultExecutorHeartbeatService.scala index 067e0d2cbb..ea3248ba6d 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/DefaultExecutorHeartbeatService.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/DefaultExecutorHeartbeatService.scala @@ -78,6 +78,7 @@ class DefaultExecutorHeartbeatService heartbeatTime, TimeUnit.MILLISECONDS ) + ExecutorHeartbeatServiceHolder.registerHeartBeatService(this) } /** diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/DefaultOperateService.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/DefaultOperateService.scala index df34626c20..c0ef50636d 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/DefaultOperateService.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/DefaultOperateService.scala @@ -18,6 +18,7 @@ package org.apache.linkis.engineconn.acessible.executor.service import org.apache.linkis.common.utils.{Logging, Utils} +import org.apache.linkis.engineconn.acessible.executor.hook.OperationHook import org.apache.linkis.manager.common.operator.OperatorFactory import org.apache.linkis.manager.common.protocol.engine.{ EngineOperateRequest, @@ -38,19 +39,50 @@ class DefaultOperateService extends OperateService with Logging { override def executeOperation( engineOperateRequest: EngineOperateRequest ): EngineOperateResponse = { + var response: EngineOperateResponse = null + val parameters = engineOperateRequest.parameters.asScala.toMap val operator = Utils.tryCatch(OperatorFactory().getOperatorRequest(parameters)) { t => logger.error(s"Get operator failed, parameters is ${engineOperateRequest.parameters}.", t) - return EngineOperateResponse(Map.empty, true, ExceptionUtils.getRootCauseMessage(t)) + response = EngineOperateResponse(Map.empty, true, ExceptionUtils.getRootCauseMessage(t)) + doPostHook(engineOperateRequest, response) + return response } logger.info( s"Try to execute operator ${operator.getClass.getSimpleName} with parameters ${engineOperateRequest.parameters}." ) val result = Utils.tryCatch(operator(parameters)) { t => logger.error(s"Execute ${operator.getClass.getSimpleName} failed.", t) - return EngineOperateResponse(Map.empty, true, ExceptionUtils.getRootCauseMessage(t)) + response = EngineOperateResponse(Map.empty, true, ExceptionUtils.getRootCauseMessage(t)) + doPostHook(engineOperateRequest, response) + return response + } + logger.info(s"End to execute operator ${operator.getClass.getSimpleName}.") + response = EngineOperateResponse(result) + doPostHook(engineOperateRequest, response) + response + } + + private def doPreHook( + engineOperateRequest: EngineOperateRequest, + engineOperateResponse: EngineOperateResponse + ): Unit = { + Utils.tryAndWarn { + OperationHook + .getOperationHooks() + .foreach(hook => hook.doPreOperation(engineOperateRequest, engineOperateResponse)) + } + } + + private def doPostHook( + engineOperateRequest: EngineOperateRequest, + engineOperateResponse: EngineOperateResponse + ): Unit = { + Utils.tryAndWarn { + OperationHook + .getOperationHooks() + .foreach(hook => hook.doPostOperation(engineOperateRequest, engineOperateResponse)) } - EngineOperateResponse(result) } } diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/EngineConnTimedLockService.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/EngineConnTimedLockService.scala index 452c6305b0..026234e938 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/EngineConnTimedLockService.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/EngineConnTimedLockService.scala @@ -51,7 +51,7 @@ class EngineConnTimedLockService extends LockService with Logging { private var lockType: EngineLockType = EngineLockType.Timed private def isSupportParallelism: Boolean = - AccessibleExecutorConfiguration.ENGINECONN_SUPPORT_PARALLELISM + AccessibleExecutorConfiguration.ENGINECONN_SUPPORT_PARALLELISM.getHotValue() /** * @param lock @@ -161,9 +161,7 @@ class EngineConnTimedLockService extends LockService with Logging { .toString ) if (isLockExist(lock)) { - logger.info( - s"try to unlock lockEntity : lockString=$lockString,lockedBy=${engineConnLock.lockedBy.getId}" - ) + logger.info(s"try to unlock lockEntity : lockString=$lockString") engineConnLock.release() this.lockString = null true diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/ExecutorHeartbeatService.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/ExecutorHeartbeatService.scala index bfecf73252..7abcbe8dcf 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/ExecutorHeartbeatService.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/ExecutorHeartbeatService.scala @@ -34,3 +34,14 @@ trait ExecutorHeartbeatService { def dealNodeHeartbeatRequest(nodeHeartbeatRequest: NodeHeartbeatRequest): NodeHeartbeatMsg } + +object ExecutorHeartbeatServiceHolder { + + private var executorHeartbeatService: ExecutorHeartbeatService = _ + + def registerHeartBeatService(executorHeartbeatService: ExecutorHeartbeatService): Unit = + this.executorHeartbeatService = executorHeartbeatService + + def getDefaultHeartbeatService(): ExecutorHeartbeatService = executorHeartbeatService + +} diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/callback/hook/CallbackEngineConnHook.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/callback/hook/CallbackEngineConnHook.scala index d7ad2c7979..a5bf02f84f 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/callback/hook/CallbackEngineConnHook.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/callback/hook/CallbackEngineConnHook.scala @@ -20,6 +20,7 @@ package org.apache.linkis.engineconn.callback.hook import org.apache.linkis.DataWorkCloudApplication import org.apache.linkis.common.conf.DWCArgumentsParser import org.apache.linkis.common.utils.{Logging, Utils} +import org.apache.linkis.engineconn.acessible.executor.entity.AccessibleExecutor import org.apache.linkis.engineconn.callback.service.{ EngineConnAfterStartCallback, EngineConnPidCallback @@ -28,6 +29,7 @@ import org.apache.linkis.engineconn.common.conf.EngineConnConf import org.apache.linkis.engineconn.common.creation.EngineCreationContext import org.apache.linkis.engineconn.common.engineconn.EngineConn import org.apache.linkis.engineconn.common.hook.EngineConnHook +import org.apache.linkis.engineconn.core.executor.ExecutorManager import org.apache.linkis.engineconn.core.hook.ShutdownHook import org.apache.linkis.manager.common.entity.enumeration.NodeStatus import org.apache.linkis.manager.common.protocol.engine.EngineConnStatusCallback @@ -59,7 +61,7 @@ class CallbackEngineConnHook extends EngineConnHook with Logging { newMap.put("spring.mvc.servlet.path", ServerConfiguration.BDP_SERVER_RESTFUL_URI.getValue) DataWorkCloudApplication.main(DWCArgumentsParser.formatSpringOptions(newMap.toMap)) - val engineConnPidCallBack = new EngineConnPidCallback(engineCreationContext.getEMInstance) + val engineConnPidCallBack = new EngineConnPidCallback() Utils.tryAndError(engineConnPidCallBack.callback()) logger.info("<--------------------SpringBoot App init succeed-------------------->") } @@ -78,9 +80,7 @@ class CallbackEngineConnHook extends EngineConnHook with Logging { engineCreationContext: EngineCreationContext, throwable: Throwable ): Unit = { - val engineConnAfterStartCallback = new EngineConnAfterStartCallback( - engineCreationContext.getEMInstance - ) + val engineConnAfterStartCallback = new EngineConnAfterStartCallback val prefixMsg = Sender.getThisServiceInstance + s": log dir: ${EngineConnConf.getLogDir}," Utils.tryAndError( engineConnAfterStartCallback.callback( @@ -88,7 +88,7 @@ class CallbackEngineConnHook extends EngineConnHook with Logging { Sender.getThisServiceInstance, engineCreationContext.getTicketId, NodeStatus.Failed, - prefixMsg + ExceptionUtils.getRootCauseMessage(throwable) + prefixMsg + ExceptionUtils.getStackTrace(throwable) ) ) ) @@ -99,15 +99,29 @@ class CallbackEngineConnHook extends EngineConnHook with Logging { protected def getNodeStatusOfStartSuccess( engineCreationContext: EngineCreationContext, engineConn: EngineConn - ): NodeStatus = NodeStatus.Success + ): NodeStatus = { + ExecutorManager.getInstance.getReportExecutor match { + case executor: AccessibleExecutor => + if ( + executor.getStatus == NodeStatus.ShuttingDown || executor.getStatus == NodeStatus.Failed + ) { + logger.info( + "The status of EngineConn is {}, and the actual status will be reported", + executor.getStatus + ) + executor.getStatus + } else { + NodeStatus.Unlock + } + case _ => NodeStatus.Unlock + } + } override def afterEngineServerStartSuccess( engineCreationContext: EngineCreationContext, engineConn: EngineConn ): Unit = { - val engineConnAfterStartCallback = new EngineConnAfterStartCallback( - engineCreationContext.getEMInstance - ) + val engineConnAfterStartCallback = new EngineConnAfterStartCallback Utils.tryAndError( engineConnAfterStartCallback.callback( EngineConnStatusCallback( diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/callback/service/EngineConnAfterStartCallback.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/callback/service/EngineConnAfterStartCallback.scala index fe6275ce67..d61e711f5d 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/callback/service/EngineConnAfterStartCallback.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/callback/service/EngineConnAfterStartCallback.scala @@ -17,10 +17,7 @@ package org.apache.linkis.engineconn.callback.service -import org.apache.linkis.common.ServiceInstance - -class EngineConnAfterStartCallback(emInstance: ServiceInstance) - extends AbstractEngineConnStartUpCallback(emInstance) { +class EngineConnAfterStartCallback extends AbstractEngineConnStartUpCallback { override def callback(): Unit = {} } diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/callback/service/EngineConnCallback.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/callback/service/EngineConnCallback.scala index 8a028d0a90..d1eb83d391 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/callback/service/EngineConnCallback.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/callback/service/EngineConnCallback.scala @@ -17,8 +17,8 @@ package org.apache.linkis.engineconn.callback.service -import org.apache.linkis.common.ServiceInstance import org.apache.linkis.common.utils.Logging +import org.apache.linkis.governance.common.conf.GovernanceCommonConf import org.apache.linkis.manager.common.entity.enumeration.NodeStatus import org.apache.linkis.manager.common.protocol.engine.EngineConnStatusCallback import org.apache.linkis.protocol.message.RequestProtocol @@ -26,31 +26,25 @@ import org.apache.linkis.rpc.Sender trait EngineConnCallback { - protected def getEMSender: Sender - def callback(): Unit } -abstract class AbstractEngineConnStartUpCallback(emInstance: ServiceInstance) - extends EngineConnCallback - with Logging { - - override protected def getEMSender: Sender = { - Sender.getSender(emInstance) - } +abstract class AbstractEngineConnStartUpCallback() extends EngineConnCallback with Logging { def callback(protocol: RequestProtocol): Unit = { protocol match { case protocol: EngineConnStatusCallback => if (protocol.status.equals(NodeStatus.Failed)) { - logger.error(s"protocol will send to em: ${protocol}") + logger.error(s"EngineConn Start Failed protocol will send to LM: ${protocol}") } else { - logger.info(s"protocol will send to em: ${protocol}") + logger.info(s"protocol will send to lm: ${protocol}") } case _ => } - getEMSender.send(protocol) + Sender + .getSender(GovernanceCommonConf.ENGINE_APPLICATION_MANAGER_SPRING_NAME.getValue) + .send(protocol) } } diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/callback/service/EngineConnPidCallback.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/callback/service/EngineConnPidCallback.scala index 23a3f90a2b..f0995c0b99 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/callback/service/EngineConnPidCallback.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/callback/service/EngineConnPidCallback.scala @@ -17,15 +17,13 @@ package org.apache.linkis.engineconn.callback.service -import org.apache.linkis.common.ServiceInstance import org.apache.linkis.engineconn.core.EngineConnObject import org.apache.linkis.governance.common.protocol.task.ResponseEngineConnPid import org.apache.linkis.rpc.Sender import java.lang.management.ManagementFactory -class EngineConnPidCallback(emInstance: ServiceInstance) - extends AbstractEngineConnStartUpCallback(emInstance) { +class EngineConnPidCallback extends AbstractEngineConnStartUpCallback { override def callback(): Unit = { val pid = ManagementFactory.getRuntimeMXBean.getName.split("@")(0) diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-plugin-core/src/main/java/org/apache/linkis/manager/engineplugin/errorcode/EngineconnCoreErrorCodeSummary.java b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-plugin-core/src/main/java/org/apache/linkis/manager/engineplugin/errorcode/EngineconnCoreErrorCodeSummary.java index 1685f4b652..24c8b904cd 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-plugin-core/src/main/java/org/apache/linkis/manager/engineplugin/errorcode/EngineconnCoreErrorCodeSummary.java +++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-plugin-core/src/main/java/org/apache/linkis/manager/engineplugin/errorcode/EngineconnCoreErrorCodeSummary.java @@ -23,7 +23,7 @@ public enum EngineconnCoreErrorCodeSummary implements LinkisErrorCode { FAILED_CREATE_ELR(10001, "Failed to createEngineConnLaunchRequest(创建 EngineConnLaunchRequest失败)"), EN_PLUGIN_MATERIAL_SOURCE_EXCEPTION( 10001, - "The engine plug-in material is abnormal, please check whether the material is uploaded successfully(引擎插件物料异常，请检查物料是否上传成功)"), + "The engine plugin material is abnormal, please check whether the material is uploaded successfully(引擎插件物料异常，请检查物料是否上传成功)"), ETL_REQUESTED(10001, "EngineTypeLabel are requested(需要参数 EngineTypeLabel)"), CANNOT_INSTANCE_ECE(20000, "Cannot instance EngineConnExecution(无法实例化 EngineConnExecution)"), @@ -32,7 +32,7 @@ public enum EngineconnCoreErrorCodeSummary implements LinkisErrorCode { UCL_NOT_EXISTS(20000, "UserCreatorLabel does not exist(UserCreatorLabel 不存在)"), CANNOT_HOME_PATH_EC(20001, "Cannot find the home path of engineConn(找不到 engineConn 的 home 路径)"), CANNOT_HOME_PATH_DIST( - 20001, "Cannot find the home path of engineconn dist(找不到 engineconn dist 的 home 路径)"), + 20001, "Cannot find the home path:{0} of engineconn dist(找不到 engineconn dist 的 home 路径)"), DIST_IS_EMPTY( 20001, "The dist of EngineConn is empty,engineConnType is:{0}(EngineConn 的 dist 为空,engineConnType为：{0})"), diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-plugin-core/src/main/scala/org/apache/linkis/manager/engineplugin/common/conf/EnvConfiguration.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-plugin-core/src/main/scala/org/apache/linkis/manager/engineplugin/common/conf/EnvConfiguration.scala index 290c6211e1..f3235ffa34 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-plugin-core/src/main/scala/org/apache/linkis/manager/engineplugin/common/conf/EnvConfiguration.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-plugin-core/src/main/scala/org/apache/linkis/manager/engineplugin/common/conf/EnvConfiguration.scala @@ -41,13 +41,12 @@ object EnvConfiguration { val ENGINE_CONN_CLASSPATH_FILES = CommonVars("wds.linkis.engineConn.files", "", "engineConn额外的配置文件") - val metaspaceSize = if (SystemUtils.isJavaVersionAtLeast(JavaVersion.JAVA_1_8)) { - "-XX:MaxMetaspaceSize=256m -XX:MetaspaceSize=128m" - } else { - "-XX:MaxPermSize=256m -XX:PermSize=128m" - } + val MAX_METASPACE_SIZE = CommonVars("linkis.engineconn.metaspace.size.max", "256m") - val ENGINE_CONN_DEFAULT_JAVA_OPTS = CommonVars[String]( + lazy val metaspaceSize = + s"-XX:MaxMetaspaceSize=${MAX_METASPACE_SIZE.getValue} -XX:MetaspaceSize=128m" + + lazy val ENGINE_CONN_DEFAULT_JAVA_OPTS = CommonVars[String]( "wds.linkis.engineConn.javaOpts.default", s"-XX:+UseG1GC ${metaspaceSize} " + s"-Xloggc:%s -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Dwds.linkis.server.conf=linkis-engineconn.properties -Dwds.linkis.gateway.url=${Configuration.getGateWayURL()}" diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-plugin-core/src/main/scala/org/apache/linkis/manager/engineplugin/common/launch/process/JavaProcessEngineConnLaunchBuilder.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-plugin-core/src/main/scala/org/apache/linkis/manager/engineplugin/common/launch/process/JavaProcessEngineConnLaunchBuilder.scala index 5271ec37e9..082b02a020 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-plugin-core/src/main/scala/org/apache/linkis/manager/engineplugin/common/launch/process/JavaProcessEngineConnLaunchBuilder.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-plugin-core/src/main/scala/org/apache/linkis/manager/engineplugin/common/launch/process/JavaProcessEngineConnLaunchBuilder.scala @@ -53,7 +53,7 @@ abstract class JavaProcessEngineConnLaunchBuilder this.engineConnResourceGenerator = engineConnResourceGenerator protected def getGcLogDir(engineConnBuildRequest: EngineConnBuildRequest): String = - variable(LOG_DIRS) + "/gc.log" + variable(LOG_DIRS) + "/gc" protected def getLogDir(engineConnBuildRequest: EngineConnBuildRequest): String = s" -Dlogging.file=${EnvConfiguration.LOG4J2_XML_FILE.getValue} " + diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-plugin-core/src/main/scala/org/apache/linkis/manager/engineplugin/common/resource/UserNodeResource.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-plugin-core/src/main/scala/org/apache/linkis/manager/engineplugin/common/resource/UserNodeResource.scala index aada8caedc..02565a394b 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-plugin-core/src/main/scala/org/apache/linkis/manager/engineplugin/common/resource/UserNodeResource.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-plugin-core/src/main/scala/org/apache/linkis/manager/engineplugin/common/resource/UserNodeResource.scala @@ -34,6 +34,9 @@ class UserNodeResource extends NodeResource { private var leftResource: Resource = _ private var createTime: Date = _ private var updateTime: Date = _ + private var maxApps: Int = _ + private var numPendingApps: Int = _ + private var numActiveApps: Int = _ def getUser: String = user @@ -87,4 +90,23 @@ class UserNodeResource extends NodeResource { override def getId: Integer = id override def setId(id: Integer): Unit = this.id = id + + override def getMaxApps: Integer = maxApps + + override def setMaxApps(maxApps: Integer): Unit = { + this.maxApps = maxApps + } + + override def getNumPendingApps: Integer = numPendingApps + + override def setNumPendingApps(numPendingApps: Integer): Unit = { + this.numPendingApps = numPendingApps + } + + override def getNumActiveApps: Integer = numActiveApps + + override def setNumActiveApps(numActiveApps: Integer): Unit = { + this.numActiveApps = numActiveApps + } + } diff --git a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/conf/EntranceSpringConfiguration.java b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/conf/EntranceSpringConfiguration.java index 0bf27a68b3..86b1a91f7a 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/conf/EntranceSpringConfiguration.java +++ b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/conf/EntranceSpringConfiguration.java @@ -42,6 +42,7 @@ import org.apache.linkis.entrance.persistence.QueryPersistenceManager; import org.apache.linkis.entrance.persistence.ResultSetEngine; import org.apache.linkis.entrance.scheduler.EntranceGroupFactory; +import org.apache.linkis.entrance.scheduler.EntranceParallelConsumerManager; import org.apache.linkis.entrance.scheduler.EntranceSchedulerContext; import org.apache.linkis.orchestrator.ecm.EngineConnManagerBuilder; import org.apache.linkis.orchestrator.ecm.EngineConnManagerBuilder$; @@ -51,7 +52,6 @@ import org.apache.linkis.scheduler.executer.ExecutorManager; import org.apache.linkis.scheduler.queue.ConsumerManager; import org.apache.linkis.scheduler.queue.GroupFactory; -import org.apache.linkis.scheduler.queue.parallelqueue.ParallelConsumerManager; import org.apache.linkis.scheduler.queue.parallelqueue.ParallelScheduler; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; @@ -150,7 +150,7 @@ public EntranceInterceptor[] entranceInterceptors() { new ScalaCodeInterceptor(), new SQLLimitEntranceInterceptor(), new CommentInterceptor(), - new SetTenantLabelInterceptor(), + // new SetTenantLabelInterceptor(), new UserCreatorIPCheckInterceptor() }; } @@ -190,7 +190,7 @@ public GroupFactory groupFactory() { @Bean @ConditionalOnMissingBean public ConsumerManager consumerManager() { - return new ParallelConsumerManager( + return new EntranceParallelConsumerManager( ENTRANCE_SCHEDULER_MAX_PARALLELISM_USERS().getValue(), "EntranceJobScheduler"); } diff --git a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/errorcode/EntranceErrorCodeSummary.java b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/errorcode/EntranceErrorCodeSummary.java index 2f045a1760..f92083df0c 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/errorcode/EntranceErrorCodeSummary.java +++ b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/errorcode/EntranceErrorCodeSummary.java @@ -62,6 +62,11 @@ public enum EntranceErrorCodeSummary implements LinkisErrorCode { INVALID_RESULTSETS(20053, "Invalid resultsets, cannot use cache(结果集无效，无法使用 cache)"), SUBMITTING_QUERY_FAILED(30009, "Submitting the query failed(提交查询失败)!"), + + SUBMIT_CODE_ISEMPTY( + 30010, + "Submitting the execution code, after code preprocessing, the real execution code is empty, please check the executed code(提交的执行代码，经过预处理后为空，请检查执行的代码是否为空或则只有注解)!"), + QUERY_STATUS_FAILED(50081, "Query from jobHistory status failed(从 jobHistory 状态查询失败)"), GET_QUERY_RESPONSE(50081, "Get query response incorrectly(获取查询响应结果不正确)"), QUERY_TASKID_ERROR(50081, "Query task of taskId:{0} error(查询任务id：{}的任务出错)"), diff --git a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/persistence/QueryPersistenceManager.java b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/persistence/QueryPersistenceManager.java index 39964fdad1..44966fc1f1 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/persistence/QueryPersistenceManager.java +++ b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/persistence/QueryPersistenceManager.java @@ -23,6 +23,7 @@ import org.apache.linkis.entrance.cs.CSEntranceHelper; import org.apache.linkis.entrance.execute.EntranceJob; import org.apache.linkis.entrance.log.FlexibleErrorCodeManager; +import org.apache.linkis.governance.common.conf.GovernanceCommonConf; import org.apache.linkis.governance.common.entity.job.JobRequest; import org.apache.linkis.protocol.engine.JobProgressInfo; import org.apache.linkis.scheduler.executer.OutputExecuteResponse; @@ -169,6 +170,11 @@ private void updateJobStatus(Job job) { JobRequest jobRequest = null; if (job.isCompleted()) { job.setProgress(1); + } else if (job.getProgress() >= 1 && job instanceof EntranceJob) { + job.setProgress(GovernanceCommonConf.FAKE_PROGRESS()); + ((EntranceJob) job) + .getJobRequest() + .setProgress(String.valueOf(GovernanceCommonConf.FAKE_PROGRESS())); } try { jobRequest = this.entranceContext.getOrCreateEntranceParser().parseToJobRequest(job); diff --git a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceConsumerRestfulApi.java b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceConsumerRestfulApi.java new file mode 100644 index 0000000000..424e7ca170 --- /dev/null +++ b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceConsumerRestfulApi.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.entrance.restful; + +import org.apache.linkis.common.conf.Configuration; +import org.apache.linkis.entrance.EntranceServer; +import org.apache.linkis.scheduler.queue.ConsumerManager; +import org.apache.linkis.server.Message; +import org.apache.linkis.server.utils.ModuleUserUtils; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import javax.servlet.http.HttpServletRequest; + +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiOperation; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Api(tags = "entrance lable manager") +@RestController +@RequestMapping(path = "/entrance/operation/consumer") +public class EntranceConsumerRestfulApi { + + private EntranceServer entranceServer; + + private static final Logger logger = LoggerFactory.getLogger(EntranceConsumerRestfulApi.class); + + @Autowired + public void setEntranceServer(EntranceServer entranceServer) { + this.entranceServer = entranceServer; + } + + @ApiOperation(value = "kill-consumer", notes = "kill consumer", response = Message.class) + @RequestMapping(path = "/kill", method = RequestMethod.GET) + public Message killConsumer( + HttpServletRequest req, @RequestParam(value = "groupName") String groupName) { + String operationUser = ModuleUserUtils.getOperationUser(req, "kill consumer"); + if (Configuration.isNotAdmin(operationUser)) { + return Message.error("only admin can do this"); + } + logger.info("user {} to kill consumer {}", operationUser, groupName); + ConsumerManager consumerManager = + entranceServer + .getEntranceContext() + .getOrCreateScheduler() + .getSchedulerContext() + .getOrCreateConsumerManager(); + consumerManager.destroyConsumer(groupName); + logger.info("user {} finished to kill consumer {}", operationUser, groupName); + return Message.ok(); + } + + @ApiOperation(value = "consumer-info", notes = "list consumers info", response = Message.class) + @RequestMapping(path = "/info", method = RequestMethod.GET) + public Message countConsumer(HttpServletRequest req) { + String operationUser = ModuleUserUtils.getOperationUser(req, "kill consumer"); + if (Configuration.isNotAdmin(operationUser)) { + return Message.error("only admin can do this"); + } + ConsumerManager consumerManager = + entranceServer + .getEntranceContext() + .getOrCreateScheduler() + .getSchedulerContext() + .getOrCreateConsumerManager(); + return Message.ok().data("consumerNum", consumerManager.listConsumers().length); + } +} diff --git a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceLabelRestfulApi.java b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceLabelRestfulApi.java index 2ab457747c..8b3a466c24 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceLabelRestfulApi.java +++ b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceLabelRestfulApi.java @@ -22,6 +22,7 @@ import org.apache.linkis.manager.label.constant.LabelKeyConstant; import org.apache.linkis.manager.label.constant.LabelValueConstant; import org.apache.linkis.protocol.label.InsLabelRefreshRequest; +import org.apache.linkis.protocol.label.InsLabelRemoveRequest; import org.apache.linkis.rpc.Sender; import org.apache.linkis.server.Message; import org.apache.linkis.server.utils.ModuleUserUtils; @@ -47,6 +48,8 @@ public class EntranceLabelRestfulApi { private static final Logger logger = LoggerFactory.getLogger(EntranceLabelRestfulApi.class); + private static Boolean offlineFlag = false; + @ApiOperation(value = "update", notes = "update route label", response = Message.class) @ApiOperationSupport(ignoreParameters = {"jsonNode"}) @RequestMapping(path = "/update", method = RequestMethod.POST) @@ -72,13 +75,41 @@ public Message updateRouteLabel(HttpServletRequest req, @RequestBody JsonNode js public Message updateRouteLabel(HttpServletRequest req) { ModuleUserUtils.getOperationUser(req, "markoffline"); Map labels = new HashMap(); - logger.info("Prepare to modify the routelabel of entry to offline"); + logger.info("Prepare to modify the routelabel of entrance to offline"); labels.put(LabelKeyConstant.ROUTE_KEY, LabelValueConstant.OFFLINE_VALUE); InsLabelRefreshRequest insLabelRefreshRequest = new InsLabelRefreshRequest(); insLabelRefreshRequest.setLabels(labels); insLabelRefreshRequest.setServiceInstance(Sender.getThisServiceInstance()); InstanceLabelClient.getInstance().refreshLabelsToInstance(insLabelRefreshRequest); + synchronized (offlineFlag) { + offlineFlag = true; + } logger.info("Finished to modify the routelabel of entry to offline"); return Message.ok(); } + + @ApiOperation( + value = "backonline", + notes = "from offline status to recover", + response = Message.class) + @RequestMapping(path = "/backonline", method = RequestMethod.GET) + public Message backOnline(HttpServletRequest req) { + ModuleUserUtils.getOperationUser(req, "backonline"); + logger.info("Prepare to modify the routelabel of entrance to remove offline"); + InsLabelRemoveRequest insLabelRemoveRequest = new InsLabelRemoveRequest(); + insLabelRemoveRequest.setServiceInstance(Sender.getThisServiceInstance()); + InstanceLabelClient.getInstance().removeLabelsFromInstance(insLabelRemoveRequest); + synchronized (offlineFlag) { + offlineFlag = false; + } + logger.info("Finished to backonline"); + return Message.ok(); + } + + @ApiOperation(value = "isOnline", notes = "entrance isOnline", response = Message.class) + @RequestMapping(path = "/isOnline", method = RequestMethod.GET) + public Message isOnline(HttpServletRequest req) { + logger.info("Whether Entrance is online: {}", !offlineFlag); + return Message.ok().data("isOnline", !offlineFlag); + } } diff --git a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceMetricRestfulApi.java b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceMetricRestfulApi.java index 7d36df8fec..7b487352d5 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceMetricRestfulApi.java +++ b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceMetricRestfulApi.java @@ -20,8 +20,7 @@ import org.apache.linkis.common.conf.Configuration; import org.apache.linkis.entrance.EntranceServer; import org.apache.linkis.entrance.execute.EntranceJob; -import org.apache.linkis.manager.label.entity.engine.EngineTypeLabel; -import org.apache.linkis.manager.label.utils.LabelUtil; +import org.apache.linkis.entrance.scheduler.CreatorECTypeDefaultConf; import org.apache.linkis.server.Message; import org.apache.linkis.server.utils.ModuleUserUtils; @@ -67,7 +66,7 @@ public Message taskinfo( HttpServletRequest req, @RequestParam(value = "user", required = false) String user, @RequestParam(value = "creator", required = false) String creator, - @RequestParam(value = "engineTypeLabel", required = false) String engineTypeLabelValue) { + @RequestParam(value = "ecType", required = false) String ecType) { String userName = ModuleUserUtils.getOperationUser(req, "taskinfo"); String queryUser = user; if (Configuration.isNotAdmin(userName)) { @@ -83,23 +82,12 @@ public Message taskinfo( } else if (StringUtils.isBlank(creator)) { filterWords = queryUser; } - EntranceJob[] undoneTasks = entranceServer.getAllUndoneTask(filterWords); - int taskNumber = 0; + EntranceJob[] undoneTasks = entranceServer.getAllUndoneTask(filterWords, ecType); int runningNumber = 0; int queuedNumber = 0; if (null != undoneTasks) { for (EntranceJob task : undoneTasks) { - if (StringUtils.isNotBlank(engineTypeLabelValue)) { - EngineTypeLabel engineTypeLabel = - LabelUtil.getEngineTypeLabel(task.getJobRequest().getLabels()); - // Task types do not match, do not count - if (null == engineTypeLabel - || !engineTypeLabelValue.equalsIgnoreCase(engineTypeLabel.getStringValue())) { - continue; - } - } - taskNumber++; if (task.isRunning()) { runningNumber++; } else { @@ -107,17 +95,25 @@ public Message taskinfo( } } } - return Message.ok("success") - .data("taskNumber", taskNumber) - .data("runningNumber", runningNumber) - .data("queuedNumber", queuedNumber); + Message resp = + Message.ok("success") + .data("taskNumber", undoneTasks.length) + .data("runningNumber", runningNumber) + .data("queuedNumber", queuedNumber); + if (StringUtils.isNoneBlank(creator, ecType)) { + int creatorECTypeMaxRunningJobs = + CreatorECTypeDefaultConf.getCreatorECTypeMaxRunningJobs(creator, ecType); + resp.data("creatorECTypeMaxRunningJobs", creatorECTypeMaxRunningJobs); + resp.data("limitExceeded", runningNumber > creatorECTypeMaxRunningJobs); + } + return resp; } - @ApiOperation(value = "Status", notes = "get running task number ", response = Message.class) + @ApiOperation(value = "runningtask", notes = "get running task number ", response = Message.class) @RequestMapping(path = "/runningtask", method = RequestMethod.GET) - public Message status(HttpServletRequest req) { + public Message runningtask(HttpServletRequest req) { ModuleUserUtils.getOperationUser(req, "runningtask"); - EntranceJob[] undoneTasks = entranceServer.getAllUndoneTask(""); + EntranceJob[] undoneTasks = entranceServer.getAllUndoneTask("", null); Boolean isCompleted = false; if (null == undoneTasks || undoneTasks.length < 1) { isCompleted = true; diff --git a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceRestfulApi.java b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceRestfulApi.java index a5d3ace35f..b616c9c658 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceRestfulApi.java +++ b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceRestfulApi.java @@ -110,15 +110,6 @@ public Message execute(HttpServletRequest req, @RequestBody Map JobRequest jobReq = ((EntranceJob) job).getJobRequest(); Long jobReqId = jobReq.getId(); ModuleUserUtils.getOperationUser(req, "execute task,id: " + jobReqId); - pushLog( - LogUtils.generateInfo( - "You have submitted a new job, script code (after variable substitution) is"), - job); - pushLog( - "************************************SCRIPT CODE************************************", job); - pushLog(jobReq.getExecutionCode(), job); - pushLog( - "************************************SCRIPT CODE************************************", job); String execID = ZuulEntranceUtils.generateExecID( job.getId(), @@ -162,15 +153,6 @@ public Message submit(HttpServletRequest req, @RequestBody Map j JobRequest jobRequest = ((EntranceJob) job).getJobRequest(); Long jobReqId = jobRequest.getId(); ModuleUserUtils.getOperationUser(req, "submit jobReqId: " + jobReqId); - pushLog( - LogUtils.generateInfo( - "You have submitted a new job, script code (after variable substitution) is"), - job); - pushLog( - "************************************SCRIPT CODE************************************", job); - pushLog(jobRequest.getExecutionCode(), job); - pushLog( - "************************************SCRIPT CODE************************************", job); pushLog( LogUtils.generateInfo( "Your job is accepted, jobID is " @@ -594,9 +576,9 @@ public Message killJobs( logger.error("kill job {} failed ", job.get().getId(), t); message = Message.error( - "An exception occurred while killing the job, kill failed(kill job的时候出现了异常，kill失败)"); + "An exception occurred while killing the job, kill failed(kill job的时候出现了异常，kill失败)", + t); message.setMethod("/api/entrance/" + id + "/kill"); - message.setStatus(1); } } messages.add(message); @@ -678,7 +660,9 @@ public Message kill( logger.error("kill job {} failed ", job.get().getId(), t); message = Message.error( - "An exception occurred while killing the job, kill failed(kill job的时候出现了异常，kill失败)"); + "An exception occurred while killing the job, kill failed(kill job的时候出现了异常，kill失败) with error:" + + t.getMessage(), + t); message.setMethod("/api/entrance/" + id + "/kill"); message.setStatus(1); } diff --git a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/scheduler/CreatorECTypeDefaultConf.java b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/scheduler/CreatorECTypeDefaultConf.java new file mode 100644 index 0000000000..5a91c71a11 --- /dev/null +++ b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/scheduler/CreatorECTypeDefaultConf.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.entrance.scheduler; + +import org.apache.linkis.common.conf.Configuration; +import org.apache.linkis.entrance.conf.EntranceConfiguration; +import org.apache.linkis.entrance.utils.EntranceUtils; +import org.apache.linkis.governance.common.protocol.conf.RequestQueryEngineConfig; +import org.apache.linkis.governance.common.protocol.conf.ResponseQueryConfig; +import org.apache.linkis.manager.label.entity.engine.EngineTypeLabel; +import org.apache.linkis.manager.label.entity.engine.UserCreatorLabel; +import org.apache.linkis.rpc.Sender; + +import org.apache.commons.lang3.StringUtils; + +import java.util.concurrent.TimeUnit; + +import scala.Tuple2; + +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class CreatorECTypeDefaultConf { + + private static final Logger logger = LoggerFactory.getLogger(CreatorECTypeDefaultConf.class); + + public static Sender confSender = + Sender.getSender( + Configuration.CLOUD_CONSOLE_CONFIGURATION_SPRING_APPLICATION_NAME().getValue()); + + private static LoadingCache confCache = + CacheBuilder.newBuilder() + .maximumSize(1000) + .expireAfterWrite( + (long) EntranceConfiguration.ENTRANCE_CREATOR_JOB_LIMIT_CONF_CACHE().getValue(), + TimeUnit.MINUTES) + .build( + new CacheLoader() { + @Override + public Integer load(String key) throws Exception { + Tuple2 tuple2 = + EntranceUtils.fromKeyGetLabels(key); + RequestQueryEngineConfig requestQueryEngineConfig = + new RequestQueryEngineConfig(tuple2._1, tuple2._2(), null); + int jobLimit = + (int) EntranceConfiguration.ENTRANCE_CREATOR_JOB_LIMIT().getValue(); + try { + Object response = confSender.ask(requestQueryEngineConfig); + if (response instanceof ResponseQueryConfig) { + jobLimit = + (int) + EntranceConfiguration.ENTRANCE_CREATOR_JOB_LIMIT() + .getValue(((ResponseQueryConfig) response).getKeyAndValue()); + } + } catch (Exception e) { + logger.warn("Failed to get key {} from conf", key, e); + } + return jobLimit; + } + }); + + public static int getCreatorECTypeMaxRunningJobs(String creator, String ecType) { + int jobLimit = (int) EntranceConfiguration.ENTRANCE_CREATOR_JOB_LIMIT().getValue(); + if (StringUtils.isNoneBlank(creator, ecType)) { + try { + String key = EntranceUtils.getDefaultCreatorECTypeKey(creator, ecType); + jobLimit = confCache.get(key); + } catch (Exception e) { + logger.warn("Failed to get key creator {} ecType {} from cache", creator, ecType, e); + } + } + int entranceNumber = EntranceUtils.getRunningEntranceNumber(); + return jobLimit / entranceNumber; + } +} diff --git a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/scheduler/EntranceFIFOUserConsumer.java b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/scheduler/EntranceFIFOUserConsumer.java new file mode 100644 index 0000000000..ac180d1aa3 --- /dev/null +++ b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/scheduler/EntranceFIFOUserConsumer.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.entrance.scheduler; + +import org.apache.linkis.scheduler.SchedulerContext; +import org.apache.linkis.scheduler.queue.Consumer; +import org.apache.linkis.scheduler.queue.Group; +import org.apache.linkis.scheduler.queue.fifoqueue.FIFOUserConsumer; + +import java.util.concurrent.ExecutorService; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class EntranceFIFOUserConsumer extends FIFOUserConsumer { + + private static final Logger logger = LoggerFactory.getLogger(EntranceFIFOUserConsumer.class); + + public EntranceFIFOUserConsumer( + SchedulerContext schedulerContext, ExecutorService executeService, Group group) { + super(schedulerContext, executeService, group); + } + + @Override + public boolean runScheduleIntercept() { + Consumer[] consumers = getSchedulerContext().getOrCreateConsumerManager().listConsumers(); + int creatorRunningJobNum = 0; + + // APP_TEST_hadoop_hive or IDE_hadoop_hive + String groupNameStr = getGroup().getGroupName(); + String[] groupNames = groupNameStr.split("_"); + int length = groupNames.length; + if (length < 3) { + return true; + } + + // APP_TEST + int lastIndex = groupNameStr.lastIndexOf("_"); + int secondLastIndex = groupNameStr.lastIndexOf("_", lastIndex - 1); + String creatorName = groupNameStr.substring(0, secondLastIndex); + + // hive + String ecType = groupNames[length - 1]; + + for (Consumer consumer : consumers) { + String groupName = consumer.getGroup().getGroupName(); + if (groupName.startsWith(creatorName) && groupName.endsWith(ecType)) { + creatorRunningJobNum += consumer.getRunningEvents().length; + } + } + + int creatorECTypeMaxRunningJobs = + CreatorECTypeDefaultConf.getCreatorECTypeMaxRunningJobs(creatorName, ecType); + + if (logger.isDebugEnabled()) { + logger.debug( + "Creator: {} EC: {} there are currently:{} jobs running and maximum limit: {}", + creatorName, + ecType, + creatorRunningJobNum, + creatorECTypeMaxRunningJobs); + } + + if (creatorRunningJobNum > creatorECTypeMaxRunningJobs) { + logger.error( + "Creator: {} EC: {} there are currently:{} jobs running that exceed the maximum limit: {}", + creatorName, + ecType, + creatorRunningJobNum, + creatorECTypeMaxRunningJobs); + return false; + } else { + return true; + } + } +} diff --git a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/scheduler/EntranceParallelConsumerManager.java b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/scheduler/EntranceParallelConsumerManager.java new file mode 100644 index 0000000000..98f0929ee9 --- /dev/null +++ b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/scheduler/EntranceParallelConsumerManager.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.entrance.scheduler; + +import org.apache.linkis.scheduler.queue.Group; +import org.apache.linkis.scheduler.queue.fifoqueue.FIFOUserConsumer; +import org.apache.linkis.scheduler.queue.parallelqueue.ParallelConsumerManager; + +public class EntranceParallelConsumerManager extends ParallelConsumerManager { + + public EntranceParallelConsumerManager(int maxParallelismUsers, String schedulerName) { + super(maxParallelismUsers, schedulerName); + } + + @Override + public FIFOUserConsumer createConsumer(String groupName) { + Group group = getSchedulerContext().getOrCreateGroupFactory().getGroup(groupName); + return new EntranceFIFOUserConsumer(getSchedulerContext(), getOrCreateExecutorService(), group); + } +} diff --git a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/DefaultEntranceServer.java b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/DefaultEntranceServer.java index 4301334965..fac85f32c1 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/DefaultEntranceServer.java +++ b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/DefaultEntranceServer.java @@ -94,11 +94,12 @@ private void shutdownEntrance(ContextClosedEvent event) { logger.warn("event has been handled"); } else { logger.warn("Entrance exit to stop all job"); - EntranceJob[] allUndoneJobs = getAllUndoneTask(null); + EntranceJob[] allUndoneJobs = getAllUndoneTask(null, null); if (null != allUndoneJobs) { for (EntranceJob job : allUndoneJobs) { job.onFailure( - "Entrance exits the automatic cleanup task and can be rerun(服务退出自动清理任务，可以重跑)", null); + "Your job will be marked as canceled because the Entrance service restarted(因为Entrance服务重启，您的任务将被标记为取消)", + null); } } } diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala index f298e54251..eed2929c23 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala @@ -20,6 +20,11 @@ package org.apache.linkis.entrance import org.apache.linkis.common.exception.{ErrorException, LinkisException, LinkisRuntimeException} import org.apache.linkis.common.log.LogUtils import org.apache.linkis.common.utils.{Logging, Utils} +import org.apache.linkis.entrance.conf.EntranceConfiguration +import org.apache.linkis.entrance.conf.EntranceConfiguration.{ + ENABLE_JOB_TIMEOUT_CHECK, + ENTRANCE_TASK_TIMEOUT +} import org.apache.linkis.entrance.cs.CSEntranceHelper import org.apache.linkis.entrance.errorcode.EntranceErrorCodeSummary._ import org.apache.linkis.entrance.exception.{EntranceErrorException, SubmitFailedException} @@ -28,6 +33,7 @@ import org.apache.linkis.entrance.log.LogReader import org.apache.linkis.entrance.timeout.JobTimeoutManager import org.apache.linkis.entrance.utils.JobHistoryHelper import org.apache.linkis.governance.common.entity.job.JobRequest +import org.apache.linkis.governance.common.utils.LoggerUtils import org.apache.linkis.protocol.constants.TaskConstant import org.apache.linkis.rpc.Sender import org.apache.linkis.scheduler.queue.{Job, SchedulerEventState} @@ -38,6 +44,7 @@ import org.apache.commons.lang3.exception.ExceptionUtils import java.text.MessageFormat import java.util +import java.util.concurrent.TimeUnit abstract class EntranceServer extends Logging { @@ -45,6 +52,8 @@ abstract class EntranceServer extends Logging { private val jobTimeoutManager: JobTimeoutManager = new JobTimeoutManager() + private val timeoutCheck = EntranceConfiguration.ENABLE_JOB_TIMEOUT_CHECK.getValue + def init(): Unit def getName: String @@ -78,12 +87,15 @@ abstract class EntranceServer extends Logging { } logger.info(s"received a request,convert $jobRequest") + LoggerUtils.setJobIdMDC(jobRequest.getId.toString) + val logAppender = new java.lang.StringBuilder() Utils.tryThrow( getEntranceContext .getOrCreateEntranceInterceptors() .foreach(int => jobRequest = int.apply(jobRequest, logAppender)) ) { t => + LoggerUtils.removeJobIdMDC() val error = t match { case error: ErrorException => error case t1: Throwable => @@ -150,6 +162,18 @@ abstract class EntranceServer extends Logging { * this to trigger JobListener.onJobinit() */ Utils.tryAndWarn(job.getJobListener.foreach(_.onJobInited(job))) + if (logger.isDebugEnabled()) { + logger.debug( + s"After code preprocessing, the real execution code is:${jobRequest.getExecutionCode}" + ) + } + if (StringUtils.isBlank(jobRequest.getExecutionCode)) { + throw new SubmitFailedException( + SUBMIT_CODE_ISEMPTY.getErrorCode, + SUBMIT_CODE_ISEMPTY.getErrorDesc + ) + } + getEntranceContext.getOrCreateScheduler().submit(job) val msg = LogUtils.generateInfo( s"Job with jobId : ${jobRequest.getId} and execID : ${job.getId()} submitted " @@ -159,14 +183,16 @@ abstract class EntranceServer extends Logging { job match { case entranceJob: EntranceJob => entranceJob.getJobRequest.setReqId(job.getId()) - if (jobTimeoutManager.timeoutCheck && JobTimeoutManager.hasTimeoutLabel(entranceJob)) { + if (timeoutCheck && JobTimeoutManager.hasTimeoutLabel(entranceJob)) { jobTimeoutManager.add(job.getId(), entranceJob) } entranceJob.getLogListener.foreach(_.onLogUpdate(entranceJob, msg)) case _ => } + LoggerUtils.removeJobIdMDC() job } { t => + LoggerUtils.removeJobIdMDC() job.onFailure("Submitting the query failed!(提交查询失败！)", t) val _jobRequest: JobRequest = getEntranceContext.getOrCreateEntranceParser().parseToJobRequest(job) @@ -206,7 +232,7 @@ abstract class EntranceServer extends Logging { entranceWebSocketService } else None - def getAllUndoneTask(filterWords: String): Array[EntranceJob] = { + def getAllUndoneTask(filterWords: String, ecType: String = null): Array[EntranceJob] = { val consumers = getEntranceContext .getOrCreateScheduler() .getSchedulerContext @@ -214,7 +240,14 @@ abstract class EntranceServer extends Logging { .listConsumers() .toSet val filterConsumer = if (StringUtils.isNotBlank(filterWords)) { - consumers.filter(_.getGroup.getGroupName.contains(filterWords)) + if (StringUtils.isNotBlank(ecType)) { + consumers.filter(consumer => + consumer.getGroup.getGroupName.contains(filterWords) && consumer.getGroup.getGroupName + .contains(ecType) + ) + } else { + consumers.filter(_.getGroup.getGroupName.contains(filterWords)) + } } else { consumers } @@ -227,6 +260,40 @@ abstract class EntranceServer extends Logging { .toArray } + /** + * to check timeout task,and kill timeout task timeout: default > 48h + */ + def startTimeOutCheck(): Unit = { + Utils.defaultScheduler.scheduleAtFixedRate( + new Runnable() { + override def run(): Unit = { + Utils.tryCatch { + + val timeoutType = EntranceConfiguration.ENTRANCE_TASK_TIMEOUT.getHotValue() + logger.info(s"Start to check timeout Job, timout is ${timeoutType}") + val timeoutTime = System.currentTimeMillis() - timeoutType.toLong + getAllUndoneTask(null, null).filter(job => job.createTime < timeoutTime).foreach { + job => + job.onFailure(s"Job has run for longer than the maximum time $timeoutType", null) + } + logger.info(s"Finished to check timeout Job, timout is ${timeoutType}") + } { case t: Throwable => + logger.warn(s"TimeoutDetective Job failed. ${t.getMessage}", t) + } + } + + }, + EntranceConfiguration.ENTRANCE_TASK_TIMEOUT_SCAN.getValue.toLong, + EntranceConfiguration.ENTRANCE_TASK_TIMEOUT_SCAN.getValue.toLong, + TimeUnit.MILLISECONDS + ) + } + + if (timeoutCheck) { + logger.info("Job time check is enabled") + startTimeOutCheck() + } + } object EntranceServer { diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceWebSocketService.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceWebSocketService.scala index 714b9f0cc2..b5339c9e2e 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceWebSocketService.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceWebSocketService.scala @@ -215,18 +215,6 @@ class EntranceWebSocketService s"Your job's execution code is (after variable substitution and code check) " ) ) - entranceServer.getEntranceContext - .getOrCreateLogManager() - .onLogUpdate( - job, - "************************************SCRIPT CODE************************************" - ) - entranceServer.getEntranceContext - .getOrCreateLogManager() - .onLogUpdate( - job, - "************************************SCRIPT CODE************************************" - ) entranceServer.getEntranceContext .getOrCreateLogManager() .onLogUpdate( diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/cache/GlobalConfigurationKeyValueCache.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/cache/GlobalConfigurationKeyValueCache.scala deleted file mode 100644 index d4190d6ca4..0000000000 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/cache/GlobalConfigurationKeyValueCache.scala +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.entrance.cache - -import org.apache.linkis.common.conf.Configuration -import org.apache.linkis.governance.common.entity.job.JobRequest -import org.apache.linkis.governance.common.protocol.conf.{ - RequestQueryGlobalConfig, - ResponseQueryConfig -} -import org.apache.linkis.protocol.CacheableProtocol -import org.apache.linkis.rpc.RPCMapCache - -import java.util - -object GlobalConfigurationKeyValueCache - extends RPCMapCache[JobRequest, String, String]( - Configuration.CLOUD_CONSOLE_CONFIGURATION_SPRING_APPLICATION_NAME.getValue - ) { - - override protected def createRequest(jobReq: JobRequest): CacheableProtocol = - RequestQueryGlobalConfig(jobReq.getExecuteUser) - - override protected def createMap(any: Any): util.Map[String, String] = any match { - case response: ResponseQueryConfig => response.getKeyAndValue - } - -} diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/conf/EntranceConfiguration.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/conf/EntranceConfiguration.scala index 5c61ce0b3b..4527bc0870 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/conf/EntranceConfiguration.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/conf/EntranceConfiguration.scala @@ -220,7 +220,29 @@ object EntranceConfiguration { val CREATOR_IP_SWITCH = CommonVars("wds.linkis.entrance.user.creator.ip.interceptor.switch", false) + val TEMPLATE_CONF_SWITCH = + CommonVars("wds.linkis.entrance.template.conf.interceptor.switch", true) + val ENABLE_ENTRANCE_DIRTY_DATA_CLEAR = CommonVars("linkis.entrance.auto.clean.dirty.data.enable", true) + val ENTRANCE_CREATOR_JOB_LIMIT: CommonVars[Int] = + CommonVars[Int]( + "linkis.entrance.creator.job.concurrency.limit", + 10000, + "Creator task concurrency limit parameters" + ) + + val ENTRANCE_CREATOR_JOB_LIMIT_CONF_CACHE = + CommonVars("linkis.entrance.creator.job.concurrency.limit.conf.cache.time", 30L) + + val ENTRANCE_TASK_TIMEOUT = + CommonVars("wds.linkis.entrance.task.timeout", new TimeType("48h")) + + val ENTRANCE_TASK_TIMEOUT_SCAN = + CommonVars("wds.linkis.entrance.task.timeout.scan", new TimeType("12h")) + + val ENABLE_HDFS_JVM_USER = + CommonVars[Boolean]("linkis.entrance.enable.hdfs.jvm.user", false).getValue + } diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/execute/DefaultEntranceExecutor.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/execute/DefaultEntranceExecutor.scala index c509f10056..266de6eb5b 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/execute/DefaultEntranceExecutor.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/execute/DefaultEntranceExecutor.scala @@ -25,6 +25,7 @@ import org.apache.linkis.entrance.orchestrator.EntranceOrchestrationFactory import org.apache.linkis.entrance.utils.JobHistoryHelper import org.apache.linkis.governance.common.entity.ExecutionNodeStatus import org.apache.linkis.governance.common.protocol.task.ResponseTaskStatus +import org.apache.linkis.governance.common.utils.LoggerUtils import org.apache.linkis.manager.label.entity.Label import org.apache.linkis.manager.label.entity.engine.CodeLanguageLabel import org.apache.linkis.manager.label.utils.LabelUtil @@ -106,6 +107,7 @@ class DefaultEntranceExecutor(id: Long) entranceExecuteRequest: EntranceExecuteRequest, orchestration: Orchestration ): Unit = { + LoggerUtils.setJobIdMDC(getId.toString) orchestrationResponse match { case succeedResponse: SucceedTaskResponse => succeedResponse match { @@ -184,6 +186,7 @@ class DefaultEntranceExecutor(id: Long) _.onLogUpdate(entranceExecuteRequest.getJob, LogUtils.generateERROR(msg)) ) } + LoggerUtils.removeJobIdMDC() } def requestToComputationJobReq(entranceExecuteRequest: EntranceExecuteRequest): JobReq = { @@ -238,11 +241,13 @@ class DefaultEntranceExecutor(id: Long) } override def kill(): Boolean = { + LoggerUtils.setJobIdMDC(getId.toString) logger.info("Entrance start to kill job {} invoke Orchestrator ", this.getId) Utils.tryAndWarn { val msg = s"You job with id was cancelled by user!" getRunningOrchestrationFuture.foreach(_.cancel(msg)) } + LoggerUtils.removeJobIdMDC() true } diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/execute/EntranceExecutor.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/execute/EntranceExecutor.scala index 44cb3620ce..be7fb13871 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/execute/EntranceExecutor.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/execute/EntranceExecutor.scala @@ -19,9 +19,11 @@ package org.apache.linkis.entrance.execute import org.apache.linkis.common.log.LogUtils import org.apache.linkis.common.utils.{Logging, Utils} +import org.apache.linkis.entrance.exception.{EntranceErrorCode, EntranceErrorException} import org.apache.linkis.entrance.job.EntranceExecuteRequest import org.apache.linkis.governance.common.entity.ExecutionNodeStatus._ import org.apache.linkis.governance.common.protocol.task.{RequestTask, ResponseTaskStatus} +import org.apache.linkis.governance.common.utils.LoggerUtils import org.apache.linkis.orchestrator.computation.operation.log.LogProcessor import org.apache.linkis.orchestrator.computation.operation.progress.ProgressProcessor import org.apache.linkis.orchestrator.computation.operation.resource.ResourceReportProcessor @@ -29,6 +31,7 @@ import org.apache.linkis.orchestrator.core.OrchestrationFuture import org.apache.linkis.protocol.UserWithCreator import org.apache.linkis.scheduler.executer._ import org.apache.linkis.scheduler.executer.ExecutorState.ExecutorState +import org.apache.linkis.server.BDPJettyServerHelper import org.apache.commons.io.IOUtils import org.apache.commons.lang3.StringUtils @@ -69,9 +72,14 @@ abstract class EntranceExecutor(val id: Long) extends Executor with Logging { } override def execute(executeRequest: ExecuteRequest): ExecuteResponse = { - var request: RequestTask = null - interceptors.foreach(in => request = in.apply(request, executeRequest)) - callExecute(executeRequest) + LoggerUtils.setJobIdMDC(getId.toString) + Utils.tryFinally { + var request: RequestTask = null + interceptors.foreach(in => request = in.apply(request, executeRequest)) + callExecute(executeRequest) + } { + LoggerUtils.removeJobIdMDC() + } } protected def callback(): Unit = {} diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/execute/EntranceExecutorManager.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/execute/EntranceExecutorManager.scala index 0d5d605983..05bc5311b0 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/execute/EntranceExecutorManager.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/execute/EntranceExecutorManager.scala @@ -90,7 +90,7 @@ abstract class EntranceExecutorManager(groupFactory: GroupFactory) job.getJobRequest match { case jobReq: JobRequest => val entranceEntranceExecutor = - new DefaultEntranceExecutor(idGenerator.incrementAndGet()) + new DefaultEntranceExecutor(jobReq.getId) // getEngineConn Executor job.getLogListener.foreach( _.onLogUpdate( diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/interceptor/impl/CommentInterceptor.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/interceptor/impl/CommentInterceptor.scala index bbb904c6a7..69b96cfe24 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/interceptor/impl/CommentInterceptor.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/interceptor/impl/CommentInterceptor.scala @@ -103,6 +103,22 @@ object ScalaCommentHelper extends CommentHelper { private val scalaCommentPattern: String = "(?ms)([\"'](?:|[^'])*['\"])|//.*?$|/\\*.*?\\*/" override def dealComment(code: String): String = code +// private val logger: Logger = LoggerFactory.getLogger(getClass) +// +// override def dealComment(code: String): String = { +// try { +// val p = Pattern.compile(scalaCommentPattern) +// val sql = p.matcher(code).replaceAll("$1") +// sql +// } catch { +// case e: Exception => +// logger.warn("scala comment failed") +// code +// case t: Throwable => +// logger.warn("scala comment failed") +// code +// } +// } } object CommentMain { diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/interceptor/impl/Explain.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/interceptor/impl/Explain.scala index 1914730d51..1f402c907d 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/interceptor/impl/Explain.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/interceptor/impl/Explain.scala @@ -133,6 +133,8 @@ object SQLExplain extends Explain { .generateWarn("please pay attention ,SQL full export mode opened(请注意,SQL全量导出模式打开)\n") ) } + var isFirstTimePrintingLimit = true + var isFirstTimePrintingOverLimit = true if (tempCode.contains("""\;""")) { val semicolonIndexes = findRealSemicolonIndex(tempCode) var oldIndex = 0 @@ -142,21 +144,27 @@ object SQLExplain extends Explain { if (isSelectCmd(singleCode)) { val trimCode = singleCode.trim if (isSelectCmdNoLimit(trimCode) && !isNoLimitAllowed) { - logAppender.append( - LogUtils.generateWarn( - s"You submitted a sql without limit, DSS will add limit 5000 to your sql" - ) + "\n" - ) + if (isFirstTimePrintingLimit) { + logAppender.append( + LogUtils.generateWarn( + s"You submitted a sql without limit, DSS will add limit 5000 to your sql" + ) + "\n" + ) + isFirstTimePrintingLimit = false + } // 将注释先干掉,然后再进行添加limit val realCode = cleanComment(trimCode) fixedCode += (realCode + SQL_APPEND_LIMIT) } else if (isSelectOverLimit(singleCode) && !isNoLimitAllowed) { val trimCode = singleCode.trim - logAppender.append( - LogUtils.generateWarn( - s"You submitted a sql with limit exceeding 5000, it is not allowed. DSS will change your limit to 5000" - ) + "\n" - ) + if (isFirstTimePrintingOverLimit) { + logAppender.append( + LogUtils.generateWarn( + s"You submitted a sql with limit exceeding 5000, it is not allowed. DSS will change your limit to 5000" + ) + "\n" + ) + isFirstTimePrintingOverLimit = false + } fixedCode += repairSelectOverLimit(trimCode) } else { fixedCode += singleCode.trim @@ -170,21 +178,27 @@ object SQLExplain extends Explain { if (isSelectCmd(singleCode)) { val trimCode = singleCode.trim if (isSelectCmdNoLimit(trimCode) && !isNoLimitAllowed) { - logAppender.append( - LogUtils.generateWarn( - s"You submitted a sql without limit, DSS will add limit 5000 to your sql" - ) + "\n" - ) + if (isFirstTimePrintingLimit) { + logAppender.append( + LogUtils.generateWarn( + s"You submitted a sql without limit, DSS will add limit 5000 to your sql" + ) + "\n" + ) + isFirstTimePrintingLimit = false + } // 将注释先干掉,然后再进行添加limit val realCode = cleanComment(trimCode) fixedCode += (realCode + SQL_APPEND_LIMIT) } else if (isSelectOverLimit(singleCode) && !isNoLimitAllowed) { val trimCode = singleCode.trim - logAppender.append( - LogUtils.generateWarn( - s"You submitted a sql with limit exceeding 5000, it is not allowed. DSS will change your limit to 5000" - ) + "\n" - ) + if (isFirstTimePrintingOverLimit) { + logAppender.append( + LogUtils.generateWarn( + s"You submitted a sql with limit exceeding 5000, it is not allowed. DSS will change your limit to 5000" + ) + "\n" + ) + isFirstTimePrintingOverLimit = false + } fixedCode += repairSelectOverLimit(trimCode) } else { fixedCode += singleCode.trim diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/observer/listener/TriggerEventListener.java b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/interceptor/impl/TemplateConfInterceptor.scala similarity index 58% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/observer/listener/TriggerEventListener.java rename to linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/interceptor/impl/TemplateConfInterceptor.scala index 8811441d02..e59557efa0 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/observer/listener/TriggerEventListener.java +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/interceptor/impl/TemplateConfInterceptor.scala @@ -15,21 +15,22 @@ * limitations under the License. */ -package org.apache.linkis.cli.application.observer.listener; +package org.apache.linkis.entrance.interceptor.impl -import org.apache.linkis.cli.application.observer.event.LinkisClientEvent; +import org.apache.linkis.entrance.conf.EntranceConfiguration +import org.apache.linkis.entrance.interceptor.EntranceInterceptor +import org.apache.linkis.governance.common.entity.job.JobRequest -import java.util.concurrent.atomic.AtomicBoolean; +import java.lang -public class TriggerEventListener implements LinkisClientListener { - AtomicBoolean atomicFlag = new AtomicBoolean(false); +class TemplateConfInterceptor extends EntranceInterceptor { - @Override - public void update(LinkisClientEvent event, Object msg) { - atomicFlag.compareAndSet(false, true); + override def apply(jobRequest: JobRequest, logAppender: lang.StringBuilder): JobRequest = { + if (EntranceConfiguration.TEMPLATE_CONF_SWITCH.getValue) { + TemplateConfUtils.dealWithStartParams(jobRequest, logAppender) + } else { + jobRequest + } } - public Boolean isTriggered() { - return atomicFlag.get(); - } } diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/interceptor/impl/TemplateConfUtils.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/interceptor/impl/TemplateConfUtils.scala new file mode 100644 index 0000000000..a3a7a3f317 --- /dev/null +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/interceptor/impl/TemplateConfUtils.scala @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.entrance.interceptor.impl + +import org.apache.linkis.common.conf.Configuration +import org.apache.linkis.common.utils.{Logging, Utils} +import org.apache.linkis.governance.common.entity.TemplateConfKey +import org.apache.linkis.governance.common.entity.job.JobRequest +import org.apache.linkis.governance.common.protocol.conf.{TemplateConfRequest, TemplateConfResponse} +import org.apache.linkis.manager.label.constant.LabelKeyConstant +import org.apache.linkis.protocol.utils.TaskUtils +import org.apache.linkis.rpc.Sender + +import org.apache.commons.lang3.StringUtils + +import java.{lang, util} +import java.util.concurrent.TimeUnit + +import scala.collection.JavaConverters._ + +import com.google.common.cache.{CacheBuilder, CacheLoader, LoadingCache} + +object TemplateConfUtils extends Logging { + + private val templateCache: LoadingCache[String, util.List[TemplateConfKey]] = CacheBuilder + .newBuilder() + .maximumSize(1000) + .expireAfterWrite(5, TimeUnit.MINUTES) + .build(new CacheLoader[String, util.List[TemplateConfKey]]() { + + override def load(templateUuid: String): util.List[TemplateConfKey] = { + var templateList = Utils.tryAndWarn { + val sender: Sender = Sender + .getSender(Configuration.CLOUD_CONSOLE_CONFIGURATION_SPRING_APPLICATION_NAME.getValue) + + logger.info(s"load template configuration data templateUuid:$templateUuid") + val res = sender.ask(new TemplateConfRequest(templateUuid)) match { + case response: TemplateConfResponse => + logger + .debug(s"${response.getList()}") + response.getList + case _ => + logger + .warn(s"load template configuration data templateUuid:$templateUuid loading failed") + new util.ArrayList[TemplateConfKey](0) + } + res + } + if (templateList.size() == 0) { + logger.warn(s"template configuration data loading failed, plaese check warn log") + } + templateList + } + + }) + + def dealWithStartParams(jobRequest: JobRequest, logAppender: lang.StringBuilder): JobRequest = { + jobRequest match { + case requestPersistTask: JobRequest => + val params = requestPersistTask.getParams + val startMap = TaskUtils.getStartupMap(params) + logger.info("jobRequest startMap params :{} ", startMap) + val templateUuid = startMap.getOrDefault(LabelKeyConstant.TEMPLATE_CONF_KEY, "").toString + if (StringUtils.isBlank(templateUuid)) { + logger.debug("jobRequest startMap param template id is empty") + } else { + logger.info("try to get template conf list with templateUid:{} ", templateUuid) + logAppender.append(s"try to get template conf list with templateUid:$templateUuid") + val templateConflist = templateCache.get(templateUuid) + if (templateConflist != null && templateConflist.size() > 0) { + val keyList = new util.HashMap[String, AnyRef]() + templateConflist.asScala.foreach(ele => { + val key = ele.getKey + val oldValue = startMap.get(key) + if (oldValue != null && StringUtils.isNotBlank(oldValue.toString)) { + logger.info(s"key:$key value:$oldValue not empty, skip to deal") + } else { + val newValue = ele.getConfigValue + logger.info(s"key:$key value:$newValue will add to startMap params") + if (TaskUtils.isWithDebugInfo(params)) { + logAppender.append(s"add $key=$newValue\n") + } + keyList.put(key, newValue) + } + + }) + if (keyList.size() > 0) { + TaskUtils.addStartupMap(params, keyList) + } + } + + } + case _ => + } + jobRequest + } + +} diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/interceptor/impl/TenantLabelSetUtils.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/interceptor/impl/TenantLabelSetUtils.scala index 1deee62a72..49f44edbae 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/interceptor/impl/TenantLabelSetUtils.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/interceptor/impl/TenantLabelSetUtils.scala @@ -95,6 +95,11 @@ object TenantLabelSetUtils extends Logging { "*-" + LabelUtil.getUserCreatorLabel(jobRequest.getLabels).getCreator.toLowerCase() ) } + if (StringUtils.isBlank(tenant)) { + tenant = userCreatorTenantCache.get( + LabelUtil.getUserCreatorLabel(jobRequest.getLabels).getUser.toLowerCase() + "-*" + ) + } logger.info("get cache tenant:" + tenant + ",jobRequest:" + jobRequest.getId) // Add cached data if it is not empty if (StringUtils.isNotBlank(tenant)) { diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/interceptor/impl/VarSubstitutionInterceptor.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/interceptor/impl/VarSubstitutionInterceptor.scala index 0487a238cf..72d40305a6 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/interceptor/impl/VarSubstitutionInterceptor.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/interceptor/impl/VarSubstitutionInterceptor.scala @@ -41,10 +41,26 @@ class VarSubstitutionInterceptor extends EntranceInterceptor { LogUtils.generateInfo("Program is substituting variables for you") + "\n" ) val codeType = LabelUtil.getCodeType(jobRequest.getLabels) - jobRequest.setExecutionCode(CustomVariableUtils.replaceCustomVar(jobRequest, codeType)) + val realCode = CustomVariableUtils.replaceCustomVar(jobRequest, codeType) + jobRequest.setExecutionCode(realCode) logAppender.append( LogUtils.generateInfo("Variables substitution ended successfully") + "\n" ) + // print code after variables substitution + logAppender.append( + LogUtils.generateInfo( + "You have submitted a new job, script code (after variable substitution) is" + ) + "\n" + ); + logAppender.append( + "************************************SCRIPT CODE************************************" + "\n" + ) + logAppender.append(realCode); + logAppender.append("\n"); + logAppender.append( + "************************************SCRIPT CODE************************************" + "\n" + ); + jobRequest } { case e: VarSubstitutionException => diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/log/CacheLogReader.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/log/CacheLogReader.scala index 483cf9ab43..748f82df4b 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/log/CacheLogReader.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/log/CacheLogReader.scala @@ -19,7 +19,11 @@ package org.apache.linkis.entrance.log import org.apache.linkis.common.io.{Fs, FsPath} import org.apache.linkis.common.utils.Utils +import org.apache.linkis.entrance.conf.EntranceConfiguration +import org.apache.linkis.entrance.exception.LogReadFailedException import org.apache.linkis.storage.FSFactory +import org.apache.linkis.storage.fs.FileSystem +import org.apache.linkis.storage.utils.StorageUtils import java.io.{InputStream, IOException} import java.util @@ -36,13 +40,26 @@ class CacheLogReader(logPath: String, charset: String, sharedCache: Cache, user: var closed = false private def createInputStream: InputStream = { + if (!logPath.contains(user)) { + throw new LogReadFailedException( + s"${user} does not have permission to read the path $logPath" + ) + } + val fsPath = new FsPath(logPath) if (fileSystem == null) lock synchronized { if (fileSystem == null) { - fileSystem = FSFactory.getFsByProxyUser(new FsPath(logPath), user) + + fileSystem = + if (StorageUtils.isHDFSPath(fsPath) && EntranceConfiguration.ENABLE_HDFS_JVM_USER) { + FSFactory.getFs(new FsPath(logPath)).asInstanceOf[FileSystem] + } else { + FSFactory.getFsByProxyUser(new FsPath(logPath), user).asInstanceOf[FileSystem] + } + fileSystem.init(new util.HashMap[String, String]()) } } - val inputStream: InputStream = fileSystem.read(new FsPath(logPath)) + val inputStream: InputStream = fileSystem.read(fsPath) inputStream } diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/log/CacheLogWriter.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/log/CacheLogWriter.scala index 9028c469ab..b54dc757cd 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/log/CacheLogWriter.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/log/CacheLogWriter.scala @@ -41,6 +41,8 @@ class CacheLogWriter(logPath: String, charset: String, sharedCache: Cache, user: val sb = new StringBuilder if (removed != null) sb.append(removed).append("\n") logs.filter(_ != null).foreach(log => sb.append(log).append("\n")) + // need append latest msg before clear + sb.append(msg).append("\n") sharedCache.cachedLogs.fakeClear() super.write(sb.toString()) pushTime.setTime( diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/log/HDFSCacheLogWriter.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/log/HDFSCacheLogWriter.scala index 24633dfbb2..a272f1e8a1 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/log/HDFSCacheLogWriter.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/log/HDFSCacheLogWriter.scala @@ -37,11 +37,15 @@ import java.util class HDFSCacheLogWriter(logPath: String, charset: String, sharedCache: Cache, user: String) extends LogWriter(charset) { - if (StringUtils.isBlank(logPath)) + if (StringUtils.isBlank(logPath)) { throw new EntranceErrorException(LOGPATH_NOT_NULL.getErrorCode, LOGPATH_NOT_NULL.getErrorDesc) + } - protected var fileSystem = + protected var fileSystem = if (EntranceConfiguration.ENABLE_HDFS_JVM_USER) { + FSFactory.getFs(new FsPath(logPath)).asInstanceOf[FileSystem] + } else { FSFactory.getFsByProxyUser(new FsPath(logPath), user).asInstanceOf[FileSystem] + } override protected var outputStream: OutputStream = null @@ -99,6 +103,8 @@ class HDFSCacheLogWriter(logPath: String, charset: String, sharedCache: Cache, u val sb = new StringBuilder if (removed != null) sb.append(removed).append("\n") logs.filter(_ != null).foreach(log => sb.append(log).append("\n")) + // need append latest msg before fake clear + sb.append(msg).append("\n") sharedCache.cachedLogs.fakeClear() writeToFile(sb.toString()) pushTime.setTime( diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/log/LoopArray.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/log/LoopArray.scala index 155d8c7bd5..5b62a49aa1 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/log/LoopArray.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/log/LoopArray.scala @@ -51,7 +51,7 @@ class LoopArray[T](maxCapacity: Int) { } else if (index > _max) { throw new IllegalArgumentException("The index " + index + " must be less than " + _max) } - val _index = (flag + (index - realSize)) % maxCapacity + val _index = (flag + (index - realSize + maxCapacity - 1)) % maxCapacity eventQueue(_index).asInstanceOf[T] } diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/orchestrator/plugin/EntranceUserParallelOrchestratorPlugin.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/orchestrator/plugin/EntranceUserParallelOrchestratorPlugin.scala index 4b9b4570f1..e5c657023e 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/orchestrator/plugin/EntranceUserParallelOrchestratorPlugin.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/orchestrator/plugin/EntranceUserParallelOrchestratorPlugin.scala @@ -20,19 +20,15 @@ package org.apache.linkis.entrance.orchestrator.plugin import org.apache.linkis.common.conf.Configuration import org.apache.linkis.common.utils.{Logging, Utils} import org.apache.linkis.entrance.conf.EntranceConfiguration +import org.apache.linkis.entrance.utils.EntranceUtils import org.apache.linkis.governance.common.protocol.conf.{ RequestQueryEngineConfigWithGlobalConfig, ResponseQueryConfig } -import org.apache.linkis.manager.label.builder.factory.LabelBuilderFactoryContext -import org.apache.linkis.manager.label.constant.LabelKeyConstant import org.apache.linkis.manager.label.entity.Label import org.apache.linkis.manager.label.entity.engine.{EngineTypeLabel, UserCreatorLabel} import org.apache.linkis.orchestrator.plugin.UserParallelOrchestratorPlugin import org.apache.linkis.rpc.Sender -import org.apache.linkis.server.BDPJettyServerHelper - -import org.apache.commons.lang3.StringUtils import java.util import java.util.concurrent.TimeUnit @@ -43,10 +39,6 @@ import com.google.common.cache.{CacheBuilder, CacheLoader, LoadingCache} class EntranceUserParallelOrchestratorPlugin extends UserParallelOrchestratorPlugin with Logging { - private val SPLIT = "," - - private val labelFactory = LabelBuilderFactoryContext.getLabelBuilderFactory - private def getDefaultMaxRuningNum: Int = { EntranceConfiguration.WDS_LINKIS_INSTANCE.getHotValue() } @@ -62,7 +54,7 @@ class EntranceUserParallelOrchestratorPlugin extends UserParallelOrchestratorPlu .build(new CacheLoader[String, Integer]() { override def load(key: String): Integer = { - val (userCreatorLabel, engineTypeLabel) = fromKeyGetLabels(key) + val (userCreatorLabel, engineTypeLabel) = EntranceUtils.fromKeyGetLabels(key) val keyAndValue = Utils.tryAndWarnMsg { sender .ask(RequestQueryEngineConfigWithGlobalConfig(userCreatorLabel, engineTypeLabel)) @@ -75,10 +67,8 @@ class EntranceUserParallelOrchestratorPlugin extends UserParallelOrchestratorPlu null == keyAndValue || !keyAndValue .containsKey(EntranceConfiguration.WDS_LINKIS_INSTANCE.key) ) { - logger.error( - s"cannot found user configuration key:${EntranceConfiguration.WDS_LINKIS_INSTANCE.key}," + - s"will use default value ${EntranceConfiguration.WDS_LINKIS_INSTANCE.getHotValue()}。All config map: ${BDPJettyServerHelper.gson - .toJson(keyAndValue)}" + logger.warn( + s"cannot found user configuration key:${EntranceConfiguration.WDS_LINKIS_INSTANCE.key}," + s"will use default value " ) } val maxRunningJobs = EntranceConfiguration.WDS_LINKIS_INSTANCE.getValue(keyAndValue, true) @@ -102,27 +92,7 @@ class EntranceUserParallelOrchestratorPlugin extends UserParallelOrchestratorPlu if (null == userCreatorLabel || null == engineTypeLabel) { return getDefaultMaxRuningNum } - configCache.get(getKey(userCreatorLabel, engineTypeLabel)) - } - - private def getKey( - userCreatorLabel: UserCreatorLabel, - engineTypeLabel: EngineTypeLabel - ): String = { - userCreatorLabel.getStringValue + SPLIT + engineTypeLabel.getStringValue - } - - private def fromKeyGetLabels(key: String): (UserCreatorLabel, EngineTypeLabel) = { - if (StringUtils.isBlank(key)) (null, null) - else { - val labelStringValues = key.split(SPLIT) - if (labelStringValues.length < 2) return (null, null) - val userCreatorLabel = labelFactory - .createLabel[UserCreatorLabel](LabelKeyConstant.USER_CREATOR_TYPE_KEY, labelStringValues(0)) - val engineTypeLabel = labelFactory - .createLabel[EngineTypeLabel](LabelKeyConstant.ENGINE_TYPE_KEY, labelStringValues(1)) - (userCreatorLabel, engineTypeLabel) - } + configCache.get(EntranceUtils.getUserCreatorEcTypeKey(userCreatorLabel, engineTypeLabel)) } override def isReady: Boolean = true diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/persistence/EntranceResultSetEngine.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/persistence/EntranceResultSetEngine.scala index 111350806e..2ba98438e8 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/persistence/EntranceResultSetEngine.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/persistence/EntranceResultSetEngine.scala @@ -17,20 +17,12 @@ package org.apache.linkis.entrance.persistence -import org.apache.linkis.common.io.{FsPath, MetaData, Record} -import org.apache.linkis.common.io.resultset.ResultSet -import org.apache.linkis.common.utils.{Logging, Utils} +import org.apache.linkis.common.utils.Logging import org.apache.linkis.entrance.exception.{EntranceErrorCode, EntranceErrorException} -import org.apache.linkis.entrance.execute.StorePathExecuteRequest -import org.apache.linkis.entrance.job.{EntranceExecuteRequest, EntranceExecutionJob} -import org.apache.linkis.entrance.scheduler.cache.CacheOutputExecuteResponse -import org.apache.linkis.governance.common.entity.job.SubJobDetail import org.apache.linkis.scheduler.executer.{AliasOutputExecuteResponse, OutputExecuteResponse} import org.apache.linkis.scheduler.queue.Job -import org.apache.linkis.storage.resultset.{ResultSetFactory, ResultSetWriter} -import org.apache.linkis.storage.utils.FileSystemUtils +import org.apache.linkis.storage.resultset.ResultSetFactory -import org.apache.commons.io.IOUtils import org.apache.commons.lang3.StringUtils class EntranceResultSetEngine extends ResultSetEngine with Logging { @@ -46,15 +38,11 @@ class EntranceResultSetEngine extends ResultSetEngine with Logging { EntranceErrorCode.RESULT_NOT_PERSISTED_ERROR.getDesc ) } - case CacheOutputExecuteResponse(alias, output) => - if (ResultSetFactory.getInstance.isResultSetPath(output)) { - getDir(output) - } else { - throw new EntranceErrorException( - EntranceErrorCode.RESULT_NOT_PERSISTED_ERROR.getErrCode, - EntranceErrorCode.RESULT_NOT_PERSISTED_ERROR.getDesc - ) - } + case _ => + throw new EntranceErrorException( + EntranceErrorCode.RESULT_NOT_PERSISTED_ERROR.getErrCode, + EntranceErrorCode.RESULT_NOT_PERSISTED_ERROR.getDesc + ) } } @@ -64,7 +52,7 @@ class EntranceResultSetEngine extends ResultSetEngine with Logging { } else { val arr = str.split("/").filter(StringUtils.isNotBlank) if (arr.length <= 2) { - return str + str } else { str.substring(0, str.lastIndexOf("/")) } diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceGroupFactory.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceGroupFactory.scala index 7f16dd2463..a436178880 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceGroupFactory.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceGroupFactory.scala @@ -17,30 +17,20 @@ package org.apache.linkis.entrance.scheduler -import org.apache.linkis.common.ServiceInstance import org.apache.linkis.common.conf.{CommonVars, Configuration} import org.apache.linkis.common.utils.{Logging, Utils} import org.apache.linkis.entrance.conf.EntranceConfiguration import org.apache.linkis.entrance.errorcode.EntranceErrorCodeSummary._ import org.apache.linkis.entrance.exception.{EntranceErrorCode, EntranceErrorException} import org.apache.linkis.entrance.execute.EntranceJob +import org.apache.linkis.entrance.utils.EntranceUtils import org.apache.linkis.governance.common.protocol.conf.{ RequestQueryEngineConfigWithGlobalConfig, ResponseQueryConfig } -import org.apache.linkis.instance.label.client.InstanceLabelClient -import org.apache.linkis.manager.label.builder.factory.LabelBuilderFactoryContext -import org.apache.linkis.manager.label.constant.{LabelKeyConstant, LabelValueConstant} import org.apache.linkis.manager.label.entity.Label -import org.apache.linkis.manager.label.entity.engine.{ - ConcurrentEngineConnLabel, - EngineTypeLabel, - UserCreatorLabel -} -import org.apache.linkis.manager.label.entity.route.RouteLabel +import org.apache.linkis.manager.label.entity.engine.{EngineTypeLabel, UserCreatorLabel} import org.apache.linkis.manager.label.utils.LabelUtil -import org.apache.linkis.protocol.constants.TaskConstant -import org.apache.linkis.protocol.utils.TaskUtils import org.apache.linkis.rpc.Sender import org.apache.linkis.scheduler.queue.{Group, GroupFactory, SchedulerEvent} import org.apache.linkis.scheduler.queue.parallelqueue.ParallelGroup @@ -51,8 +41,6 @@ import java.util import java.util.concurrent.TimeUnit import java.util.regex.Pattern -import scala.collection.JavaConverters._ - import com.google.common.cache.{Cache, CacheBuilder} class EntranceGroupFactory extends GroupFactory with Logging { @@ -81,29 +69,16 @@ class EntranceGroupFactory extends GroupFactory with Logging { } override def getOrCreateGroup(event: SchedulerEvent): Group = { - val (labels, params) = event match { + val labels = event match { case job: EntranceJob => - (job.getJobRequest.getLabels, job.getJobRequest.getParams) + job.getJobRequest.getLabels + case _ => + throw new EntranceErrorException(LABEL_NOT_NULL.getErrorCode, LABEL_NOT_NULL.getErrorDesc) } - val groupName = EntranceGroupFactory.getGroupNameByLabels(labels, params) + val groupName = EntranceGroupFactory.getGroupNameByLabels(labels) val cacheGroup = groupNameToGroups.getIfPresent(groupName) if (null == cacheGroup) synchronized { val maxAskExecutorTimes = EntranceConfiguration.MAX_ASK_EXECUTOR_TIME.getValue.toLong - if (groupName.startsWith(EntranceGroupFactory.CONCURRENT)) { - if (null == groupNameToGroups.getIfPresent(groupName)) synchronized { - if (null == groupNameToGroups.getIfPresent(groupName)) { - val group = new ParallelGroup( - groupName, - 100, - EntranceConfiguration.CONCURRENT_FACTORY_MAX_CAPACITY.getValue - ) - group.setMaxRunningJobs(EntranceConfiguration.CONCURRENT_MAX_RUNNING_JOBS.getValue) - group.setMaxAskExecutorTimes(EntranceConfiguration.CONCURRENT_EXECUTOR_TIME.getValue) - groupNameToGroups.put(groupName, group) - return group - } - } - } val sender: Sender = Sender.getSender(Configuration.CLOUD_CONSOLE_CONFIGURATION_SPRING_APPLICATION_NAME.getValue) val userCreatorLabel: UserCreatorLabel = LabelUtil.getUserCreatorLabel(labels) @@ -141,8 +116,11 @@ class EntranceGroupFactory extends GroupFactory with Logging { group.setMaxRunningJobs(maxRunningJobs) group.setMaxAskExecutorTimes(maxAskExecutorTimes) groupNameToGroups.put(groupName, group) + group + } + else { + cacheGroup } - groupNameToGroups.getIfPresent(groupName) } override def getGroup(groupName: String): Group = { @@ -156,105 +134,40 @@ class EntranceGroupFactory extends GroupFactory with Logging { group } + /** + * User task concurrency control is controlled for multiple Entrances, which will be evenly + * distributed based on the number of existing Entrances + * @param keyAndValue + * @return + */ private def getUserMaxRunningJobs(keyAndValue: util.Map[String, String]): Int = { - var userDefinedRunningJobs = EntranceConfiguration.WDS_LINKIS_INSTANCE.getValue(keyAndValue) - var entranceNum = Sender.getInstances(Sender.getThisServiceInstance.getApplicationName).length - val labelList = new util.ArrayList[Label[_]]() - val offlineRouteLabel = LabelBuilderFactoryContext.getLabelBuilderFactory - .createLabel[RouteLabel](LabelKeyConstant.ROUTE_KEY, LabelValueConstant.OFFLINE_VALUE) - labelList.add(offlineRouteLabel) - var offlineIns: Array[ServiceInstance] = null - Utils.tryAndWarn { - offlineIns = InstanceLabelClient.getInstance - .getInstanceFromLabel(labelList) - .asScala - .filter(l => - null != l && l.getApplicationName - .equalsIgnoreCase(Sender.getThisServiceInstance.getApplicationName) - ) - .toArray - } - if (null != offlineIns) { - logger.info(s"There are ${offlineIns.length} offlining instance.") - entranceNum = entranceNum - offlineIns.length - } - /* - Sender.getInstances may get 0 instances due to cache in Sender. So this instance is the one instance. - */ - if (0 >= entranceNum) { - logger.error( - s"Got ${entranceNum} ${Sender.getThisServiceInstance.getApplicationName} instances." - ) - entranceNum = 1 - } + val userDefinedRunningJobs = EntranceConfiguration.WDS_LINKIS_INSTANCE.getValue(keyAndValue) + val entranceNum = EntranceUtils.getRunningEntranceNumber() Math.max( EntranceConfiguration.ENTRANCE_INSTANCE_MIN.getValue, userDefinedRunningJobs / entranceNum - ); + ) } } object EntranceGroupFactory { - val CACHE = "_Cache" - - val CONCURRENT = "Concurrent_" - - def getGroupName( - creator: String, - user: String, - params: util.Map[String, AnyRef] = new util.HashMap[String, AnyRef] - ): String = { - val runtime = TaskUtils.getRuntimeMap(params) - val cache = - if ( - runtime.get(TaskConstant.READ_FROM_CACHE) != null && runtime - .get(TaskConstant.READ_FROM_CACHE) - .asInstanceOf[Boolean] - ) { - CACHE - } else "" - if (StringUtils.isNotEmpty(creator)) creator + "_" + user + cache - else EntranceConfiguration.DEFAULT_REQUEST_APPLICATION_NAME.getValue + "_" + user + cache - } - - def getGroupNameByLabels( - labels: java.util.List[Label[_]], - params: util.Map[String, AnyRef] = new util.HashMap[String, AnyRef] - ): String = { - - val userCreator = labels.asScala.find(_.isInstanceOf[UserCreatorLabel]) - val engineType = labels.asScala.find(_.isInstanceOf[EngineTypeLabel]) - val concurrent = labels.asScala.find(_.isInstanceOf[ConcurrentEngineConnLabel]) - if (userCreator.isEmpty || engineType.isEmpty) { + /** + * Entrance group rule creator_username_engineType eg:IDE_PEACEWONG_SPARK + * @param labels + * @param params + * @return + */ + def getGroupNameByLabels(labels: java.util.List[Label[_]]): String = { + val userCreatorLabel = LabelUtil.getUserCreatorLabel(labels) + val engineTypeLabel = LabelUtil.getEngineTypeLabel(labels) + if (null == userCreatorLabel || null == engineTypeLabel) { throw new EntranceErrorException(LABEL_NOT_NULL.getErrorCode, LABEL_NOT_NULL.getErrorDesc) } - - if (concurrent.isDefined) { - - val engineTypeLabel = engineType.get.asInstanceOf[EngineTypeLabel] - val groupName = CONCURRENT + engineTypeLabel.getEngineType - groupName - - } else { - val userCreatorLabel = userCreator.get.asInstanceOf[UserCreatorLabel] - - val engineTypeLabel = engineType.get.asInstanceOf[EngineTypeLabel] - - val runtime = TaskUtils.getRuntimeMap(params) - val cache = - if ( - runtime.get(TaskConstant.READ_FROM_CACHE) != null && runtime - .get(TaskConstant.READ_FROM_CACHE) - .asInstanceOf[Boolean] - ) { - CACHE - } else "" - val groupName = - userCreatorLabel.getCreator + "_" + userCreatorLabel.getUser + "_" + engineTypeLabel.getEngineType + cache - groupName - } + val groupName = + userCreatorLabel.getCreator + "_" + userCreatorLabel.getUser + "_" + engineTypeLabel.getEngineType + groupName } } diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/cache/CacheOutputExecuteResponse.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/cache/CacheOutputExecuteResponse.scala deleted file mode 100644 index 47a6ce9e9e..0000000000 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/cache/CacheOutputExecuteResponse.scala +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.entrance.scheduler.cache - -import org.apache.linkis.scheduler.executer.OutputExecuteResponse - -case class CacheOutputExecuteResponse(alias: String, output: String) extends OutputExecuteResponse { - override def getOutput: String = output -} diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/cache/ReadCacheConsumer.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/cache/ReadCacheConsumer.scala deleted file mode 100644 index 65bbbd39b4..0000000000 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/cache/ReadCacheConsumer.scala +++ /dev/null @@ -1,158 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.entrance.scheduler.cache - -import org.apache.linkis.common.io.FsPath -import org.apache.linkis.common.utils.Utils -import org.apache.linkis.entrance.errorcode.EntranceErrorCodeSummary._ -import org.apache.linkis.entrance.exception.CacheNotReadyException -import org.apache.linkis.entrance.execute.EntranceJob -import org.apache.linkis.entrance.persistence.PersistenceManager -import org.apache.linkis.entrance.utils.JobHistoryHelper -import org.apache.linkis.governance.common.entity.job.JobRequest -import org.apache.linkis.manager.label.constant.LabelKeyConstant -import org.apache.linkis.protocol.constants.TaskConstant -import org.apache.linkis.protocol.utils.TaskUtils -import org.apache.linkis.scheduler.SchedulerContext -import org.apache.linkis.scheduler.errorcode.LinkisSchedulerErrorCodeSummary._ -import org.apache.linkis.scheduler.exception.SchedulerErrorException -import org.apache.linkis.scheduler.executer.SuccessExecuteResponse -import org.apache.linkis.scheduler.queue.Group -import org.apache.linkis.scheduler.queue.fifoqueue.FIFOUserConsumer -import org.apache.linkis.server.BDPJettyServerHelper -import org.apache.linkis.storage.FSFactory -import org.apache.linkis.storage.fs.FileSystem - -import org.apache.commons.io.FilenameUtils -import org.apache.commons.lang3.StringUtils - -import java.util.concurrent.ExecutorService - -import scala.collection.JavaConverters._ - -import com.google.common.collect.Lists - -class ReadCacheConsumer( - schedulerContext: SchedulerContext, - executeService: ExecutorService, - private var group: Group, - persistenceManager: PersistenceManager -) extends FIFOUserConsumer(schedulerContext, executeService, group) { - - override protected def loop(): Unit = { - val event = Option(getConsumeQueue.take()) - event.foreach { - case job: EntranceJob => - job.getJobRequest match { - case jobRequest: JobRequest => - Utils.tryCatch { - val engineTpyeLabel = jobRequest.getLabels.asScala - .filter(l => l.getLabelKey.equalsIgnoreCase(LabelKeyConstant.ENGINE_TYPE_KEY)) - .headOption - .getOrElse(null) - val labelStrList = jobRequest.getLabels.asScala.map { case l => - l.getStringValue - }.toList - if (null == engineTpyeLabel) { - logger.error( - "Invalid engineType null, cannot process. jobReq : " + BDPJettyServerHelper.gson - .toJson(jobRequest) - ) - throw CacheNotReadyException( - INVALID_ENGINETYPE_NULL.getErrorCode, - INVALID_ENGINETYPE_NULL.getErrorDesc - ) - } - val readCacheBefore: Long = TaskUtils - .getRuntimeMap(job.getParams) - .getOrDefault(TaskConstant.READ_CACHE_BEFORE, 300L: java.lang.Long) - .asInstanceOf[Long] - val cacheResult = JobHistoryHelper.getCache( - jobRequest.getExecutionCode, - jobRequest.getExecuteUser, - labelStrList.asJava, - readCacheBefore - ) - if (cacheResult != null && StringUtils.isNotBlank(cacheResult.getResultLocation)) { - val resultSets = listResults(cacheResult.getResultLocation, job.getUser) - if (resultSets.size() > 0) { - for (resultSet: FsPath <- resultSets.asScala) { - val alias = FilenameUtils.getBaseName(resultSet.getPath) - val output = FsPath - .getFsPath( - cacheResult.getResultLocation, - FilenameUtils.getName(resultSet.getPath) - ) - .getSchemaPath -// persistenceManager.onResultSetCreated(job, new CacheOutputExecuteResponse(alias, output)) - throw CacheNotReadyException( - INVALID_RESULTSETS.getErrorCode, - INVALID_RESULTSETS.getErrorDesc - ) - // todo check - } -// persistenceManager.onResultSizeCreated(job, resultSets.size()) - } - val runtime = TaskUtils.getRuntimeMap(job.getParams) - runtime.put(TaskConstant.CACHE, java.lang.Boolean.FALSE) - TaskUtils.addRuntimeMap(job.getParams, runtime) - job.transitionCompleted(SuccessExecuteResponse(), "Result found in cache") - } else { - logger.info("Cache not found, submit to normal consumer.") - submitToExecute(job) - } - } { t => - logger.warn("Read cache failed, submit to normal consumer: ", t) - submitToExecute(job) - } - case _ => - } - case _ => - } - } - - private def listResults(resultLocation: String, user: String) = { - val dirPath = FsPath.getFsPath(resultLocation) - val fileSystem = FSFactory.getFsByProxyUser(dirPath, user).asInstanceOf[FileSystem] - Utils.tryFinally { - fileSystem.init(null) - if (fileSystem.exists(dirPath)) { - fileSystem.listPathWithError(dirPath).getFsPaths - } else { - Lists.newArrayList[FsPath]() - } - }(Utils.tryQuietly(fileSystem.close())) - } - - private def submitToExecute(job: EntranceJob): Unit = { - val runtime = TaskUtils.getRuntimeMap(job.getParams) - runtime.put(TaskConstant.READ_FROM_CACHE, java.lang.Boolean.FALSE) - TaskUtils.addRuntimeMap(job.getParams, runtime) - val groupName = schedulerContext.getOrCreateGroupFactory.getOrCreateGroup(job).getGroupName - val consumer = schedulerContext.getOrCreateConsumerManager.getOrCreateConsumer(groupName) - val index = consumer.getConsumeQueue.offer(job) - // index.map(getEventId(_, groupName)).foreach(job.setId) - if (index.isEmpty) { - throw new SchedulerErrorException( - JOB_QUEUE_IS_FULL.getErrorCode, - JOB_QUEUE_IS_FULL.getErrorDesc - ) - } - } - -} diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/cache/ReadCacheConsumerManager.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/cache/ReadCacheConsumerManager.scala deleted file mode 100644 index a4cba19f34..0000000000 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/cache/ReadCacheConsumerManager.scala +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.entrance.scheduler.cache - -import org.apache.linkis.entrance.persistence.PersistenceManager -import org.apache.linkis.entrance.scheduler.EntranceGroupFactory -import org.apache.linkis.scheduler.queue.fifoqueue.FIFOUserConsumer -import org.apache.linkis.scheduler.queue.parallelqueue.ParallelConsumerManager - -class ReadCacheConsumerManager(maxParallelismUsers: Int, persistenceManager: PersistenceManager) - extends ParallelConsumerManager(maxParallelismUsers) { - - override protected def createConsumer(groupName: String): FIFOUserConsumer = { - val group = getSchedulerContext.getOrCreateGroupFactory.getGroup(groupName) - if (groupName.endsWith(EntranceGroupFactory.CACHE)) { - logger.info("Create cache consumer with group: " + groupName) - new ReadCacheConsumer( - getSchedulerContext, - getOrCreateExecutorService, - group, - persistenceManager - ) - } else { - logger.info("Create normal consumer with group: " + groupName) - new FIFOUserConsumer(getSchedulerContext, getOrCreateExecutorService, group) - } - } - -} diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/timeout/JobTimeoutManager.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/timeout/JobTimeoutManager.scala index aaaf131bd8..4e62430316 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/timeout/JobTimeoutManager.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/timeout/JobTimeoutManager.scala @@ -38,8 +38,8 @@ class JobTimeoutManager extends Logging { private[this] final val timeoutJobByName: ConcurrentMap[String, EntranceJob] = new ConcurrentHashMap[String, EntranceJob] - val timeoutCheck: Boolean = EntranceConfiguration.ENABLE_JOB_TIMEOUT_CHECK.getValue - val timeoutScanInterval: Int = EntranceConfiguration.TIMEOUT_SCAN_INTERVAL.getValue + private val timeoutCheck: Boolean = EntranceConfiguration.ENABLE_JOB_TIMEOUT_CHECK.getValue + private val timeoutScanInterval: Int = EntranceConfiguration.TIMEOUT_SCAN_INTERVAL.getValue def add(jobKey: String, job: EntranceJob): Unit = { logger.info(s"Adding timeout job: ${job.getId()}") @@ -77,75 +77,75 @@ class JobTimeoutManager extends Logging { } private def timeoutDetective(): Unit = { - if (timeoutCheck) { - def checkAndSwitch(job: EntranceJob): Unit = { - logger.info(s"Checking whether the job id ${job.getJobRequest.getId()} timed out. ") - val currentTimeSeconds = System.currentTimeMillis() / 1000 - // job.isWaiting == job in queue - val jobScheduleStartTimeSeconds = - if (job.isWaiting) job.createTime / 1000 else currentTimeSeconds - val queuingTimeSeconds = currentTimeSeconds - jobScheduleStartTimeSeconds - val jobRunningStartTimeSeconds = - if (job.getStartTime > 0) job.getStartTime / 1000 else currentTimeSeconds - val runningTimeSeconds = currentTimeSeconds - jobRunningStartTimeSeconds - if (!job.isCompleted) { - job.jobRequest.getLabels.asScala foreach { - case queueTimeOutLabel: JobQueuingTimeoutLabel => - if ( - job.isWaiting && queueTimeOutLabel.getQueuingTimeout > 0 && queuingTimeSeconds >= queueTimeOutLabel.getQueuingTimeout - ) { - logger.warn( - s"Job ${job.getJobRequest.getId()} queued time : ${queuingTimeSeconds} seconds, which was over queueTimeOut : ${queueTimeOutLabel.getQueuingTimeout} seconds, cancel it now! " - ) - job.onFailure( - s"Job queued ${queuingTimeSeconds} seconds over max queue time : ${queueTimeOutLabel.getQueuingTimeout} seconds.", - null - ) - } - case jobRunningTimeoutLabel: JobRunningTimeoutLabel => - if ( - job.isRunning && jobRunningTimeoutLabel.getRunningTimeout > 0 && runningTimeSeconds >= jobRunningTimeoutLabel.getRunningTimeout - ) { - logger.warn( - s"Job ${job.getJobRequest.getId()} run timeout ${runningTimeSeconds} seconds, which was over runTimeOut : ${jobRunningTimeoutLabel.getRunningTimeout} seconds, cancel it now! " - ) - job.onFailure( - s"Job run ${runningTimeSeconds} seconds over max run time : ${jobRunningTimeoutLabel.getRunningTimeout} seconds.", - null - ) - } - case _ => - } + def checkAndSwitch(job: EntranceJob): Unit = { + logger.info(s"Checking whether the job id ${job.getJobRequest.getId()} timed out. ") + val currentTimeSeconds = System.currentTimeMillis() / 1000 + // job.isWaiting == job in queue + val jobScheduleStartTimeSeconds = + if (job.isWaiting) job.createTime / 1000 else currentTimeSeconds + val queuingTimeSeconds = currentTimeSeconds - jobScheduleStartTimeSeconds + val jobRunningStartTimeSeconds = + if (job.getStartTime > 0) job.getStartTime / 1000 else currentTimeSeconds + val runningTimeSeconds = currentTimeSeconds - jobRunningStartTimeSeconds + if (!job.isCompleted) { + job.jobRequest.getLabels.asScala foreach { + case queueTimeOutLabel: JobQueuingTimeoutLabel => + if ( + job.isWaiting && queueTimeOutLabel.getQueuingTimeout > 0 && queuingTimeSeconds >= queueTimeOutLabel.getQueuingTimeout + ) { + logger.warn( + s"Job ${job.getJobRequest.getId()} queued time : ${queuingTimeSeconds} seconds, which was over queueTimeOut : ${queueTimeOutLabel.getQueuingTimeout} seconds, cancel it now! " + ) + job.onFailure( + s"Job queued ${queuingTimeSeconds} seconds over max queue time : ${queueTimeOutLabel.getQueuingTimeout} seconds.", + null + ) + } + case jobRunningTimeoutLabel: JobRunningTimeoutLabel => + if ( + job.isRunning && jobRunningTimeoutLabel.getRunningTimeout > 0 && runningTimeSeconds >= jobRunningTimeoutLabel.getRunningTimeout + ) { + logger.warn( + s"Job ${job.getJobRequest.getId()} run timeout ${runningTimeSeconds} seconds, which was over runTimeOut : ${jobRunningTimeoutLabel.getRunningTimeout} seconds, cancel it now! " + ) + job.onFailure( + s"Job run ${runningTimeSeconds} seconds over max run time : ${jobRunningTimeoutLabel.getRunningTimeout} seconds.", + null + ) + } + case _ => } } - - timeoutJobByName.asScala.foreach(item => { - logger.info(s"Running timeout detection!") - synchronized { - jobCompleteDelete(item._1) - if (jobExist(item._1)) checkAndSwitch(item._2) - } - }) } + + timeoutJobByName.asScala.foreach(item => { + logger.info(s"Running timeout detection!") + synchronized { + jobCompleteDelete(item._1) + if (jobExist(item._1)) checkAndSwitch(item._2) + } + }) } // Thread periodic scan timeout task - val woker = Utils.defaultScheduler.scheduleAtFixedRate( - new Runnable() { - - override def run(): Unit = { - Utils.tryCatch { - timeoutDetective() - } { case t: Throwable => - logger.error(s"TimeoutDetective task failed. ${t.getMessage}", t) + if (timeoutCheck) { + val woker = Utils.defaultScheduler.scheduleAtFixedRate( + new Runnable() { + + override def run(): Unit = { + Utils.tryCatch { + timeoutDetective() + } { case t: Throwable => + logger.warn(s"TimeoutDetective task failed. ${t.getMessage}", t) + } } - } - }, - 0, - timeoutScanInterval, - TimeUnit.SECONDS - ) + }, + 0, + timeoutScanInterval, + TimeUnit.SECONDS + ) + } } diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/utils/EntranceUtils.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/utils/EntranceUtils.scala new file mode 100644 index 0000000000..13dcefa9f9 --- /dev/null +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/utils/EntranceUtils.scala @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.entrance.utils + +import org.apache.linkis.common.ServiceInstance +import org.apache.linkis.common.utils.{Logging, Utils} +import org.apache.linkis.instance.label.client.InstanceLabelClient +import org.apache.linkis.manager.label.builder.factory.LabelBuilderFactoryContext +import org.apache.linkis.manager.label.constant.{LabelKeyConstant, LabelValueConstant} +import org.apache.linkis.manager.label.entity.Label +import org.apache.linkis.manager.label.entity.engine.{EngineTypeLabel, UserCreatorLabel} +import org.apache.linkis.manager.label.entity.route.RouteLabel +import org.apache.linkis.manager.label.utils.EngineTypeLabelCreator +import org.apache.linkis.rpc.Sender + +import org.apache.commons.lang3.StringUtils + +import java.util + +import scala.collection.JavaConverters.asScalaBufferConverter + +object EntranceUtils extends Logging { + + private val SPLIT = "," + + private val labelFactory = LabelBuilderFactoryContext.getLabelBuilderFactory + + def getUserCreatorEcTypeKey( + userCreatorLabel: UserCreatorLabel, + engineTypeLabel: EngineTypeLabel + ): String = { + userCreatorLabel.getStringValue + SPLIT + engineTypeLabel.getStringValue + } + + def fromKeyGetLabels(key: String): (UserCreatorLabel, EngineTypeLabel) = { + if (StringUtils.isBlank(key)) (null, null) + else { + val labelStringValues = key.split(SPLIT) + if (labelStringValues.length < 2) return (null, null) + val userCreatorLabel = labelFactory + .createLabel[UserCreatorLabel](LabelKeyConstant.USER_CREATOR_TYPE_KEY, labelStringValues(0)) + val engineTypeLabel = labelFactory + .createLabel[EngineTypeLabel](LabelKeyConstant.ENGINE_TYPE_KEY, labelStringValues(1)) + (userCreatorLabel, engineTypeLabel) + } + } + + def getDefaultCreatorECTypeKey(creator: String, ecType: String): String = { + val userCreatorLabel = + labelFactory.createLabel[UserCreatorLabel](LabelKeyConstant.USER_CREATOR_TYPE_KEY) + val ecTypeLabel = EngineTypeLabelCreator.createEngineTypeLabel(ecType) + userCreatorLabel.setUser("*") + userCreatorLabel.setCreator(creator) + getUserCreatorEcTypeKey(userCreatorLabel, ecTypeLabel) + } + + def getRunningEntranceNumber(): Int = { + val entranceNum = Sender.getInstances(Sender.getThisServiceInstance.getApplicationName).length + val labelList = new util.ArrayList[Label[_]]() + val offlineRouteLabel = LabelBuilderFactoryContext.getLabelBuilderFactory + .createLabel[RouteLabel](LabelKeyConstant.ROUTE_KEY, LabelValueConstant.OFFLINE_VALUE) + labelList.add(offlineRouteLabel) + var offlineIns: Array[ServiceInstance] = null + Utils.tryAndWarn { + offlineIns = InstanceLabelClient.getInstance + .getInstanceFromLabel(labelList) + .asScala + .filter(l => + null != l && l.getApplicationName + .equalsIgnoreCase(Sender.getThisServiceInstance.getApplicationName) + ) + .toArray + } + val entranceRealNumber = if (null != offlineIns) { + logger.info(s"There are ${offlineIns.length} offlining instance.") + entranceNum - offlineIns.length + } else { + entranceNum + } + /* + Sender.getInstances may get 0 instances due to cache in Sender. So this instance is the one instance. + */ + if (entranceRealNumber <= 0) { + logger.error( + s"Got ${entranceRealNumber} ${Sender.getThisServiceInstance.getApplicationName} instances." + ) + 1 + } else { + entranceRealNumber + } + } + +} diff --git a/linkis-computation-governance/linkis-entrance/src/test/scala/org/apache/linkis/entrance/interceptor/impl/TestCommentHelper.java b/linkis-computation-governance/linkis-entrance/src/test/scala/org/apache/linkis/entrance/interceptor/impl/TestCommentHelper.java new file mode 100644 index 0000000000..34482d293c --- /dev/null +++ b/linkis-computation-governance/linkis-entrance/src/test/scala/org/apache/linkis/entrance/interceptor/impl/TestCommentHelper.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.entrance.interceptor.impl; + +import static org.junit.jupiter.api.Assertions.*; + +import java.util.Arrays; +import org.apache.linkis.common.utils.CodeAndRunTypeUtils; +import org.junit.jupiter.api.Test; +import org.junit.platform.commons.util.StringUtils; +import scala.reflect.internal.TypeDebugging; + +public class TestCommentHelper { + String sqlCode="" + + "--注解\n" + + "select * from table;\n" + + " --注解 \n" + + "--注解\n" + + " select \"--注解\" as test\n" + + " --@set yy=123\n" + + " --注解"; + + String scalaCode="" + + "// 注解\n" + + "print(1+1)\n" + + "//@set yy=123\n" + + " print(2)\n" + + " // 注解 \n" + + "// test\n" + + "print(\"//注解测试\")"; + + String scalaCodeRes="print(1+1)\n" + + "print(2)\n" + + "print(\"//注解测试\")"; + + @Test + void sqlDealCommentTest() { + String code=SQLCommentHelper.dealComment(sqlCode); + //System.out.println(code); + } + + @Test + void scalaDealCommentTest() { + String code=ScalaCommentHelper.dealComment(scalaCode); + String[] lines = Arrays.stream(code.split("\n")) + .map(String::trim) + .filter(x -> StringUtils.isNotBlank(x)) + .toArray(String[]::new); + String result=String.join("\n",lines); + // assertEquals(result,scalaCodeRes); + } + + +} \ No newline at end of file diff --git a/linkis-computation-governance/linkis-entrance/src/test/scala/org/apache/linkis/entrance/interceptor/impl/TestHDFSCacheLogWriter.java b/linkis-computation-governance/linkis-entrance/src/test/scala/org/apache/linkis/entrance/interceptor/impl/TestHDFSCacheLogWriter.java new file mode 100644 index 0000000000..34826ff222 --- /dev/null +++ b/linkis-computation-governance/linkis-entrance/src/test/scala/org/apache/linkis/entrance/interceptor/impl/TestHDFSCacheLogWriter.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.linkis.entrance.interceptor.impl; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.UUID; +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.linkis.cs.common.entity.source.ContextID; +import org.apache.linkis.entrance.log.Cache; +import org.apache.linkis.entrance.log.HDFSCacheLogWriter; +import org.apache.linkis.rpc.BaseRPCSender; +import org.apache.linkis.rpc.Sender; +import org.apache.linkis.storage.fs.FileSystem; +import org.apache.linkis.storage.fs.impl.LocalFileSystem; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import oshi.util.FileUtil; + +class TestHDFSCacheLogWriter { + + + @Test + void write() throws IOException { + + Cache cache = new Cache(5); + String fileName= UUID.randomUUID().toString().replace("-", "")+"-test.log"; + String logPath = System.getProperty("java.io.tmpdir")+ File.separator+fileName; + System.out.println(logPath); + String chartSet="utf-8"; + String username=System.getProperty("user.name"); + + File file=new File(logPath); + file.createNewFile(); + + HDFSCacheLogWriter logWriter =new HDFSCacheLogWriter( + //"D:\\DataSphere\\linkis\\docs\\test.log", + logPath, + chartSet, + cache, + username + ); + + String[] msgArr =new String[]{"1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18"}; + + List msgList = new ArrayList(Arrays.asList(msgArr)); + String msg=String.join("\n", msgList); + + logWriter.write(msg); + logWriter.flush(); + + List list = FileUtil.readFile(logPath); + String res=String.join("\n", list); + + res=res.replace("\n\n","\n"); + res=StringUtils.strip(res, " \n"); + Assertions.assertEquals(res,msg); + + + } +} \ No newline at end of file diff --git a/linkis-computation-governance/linkis-jdbc-driver/pom.xml b/linkis-computation-governance/linkis-jdbc-driver/pom.xml index cdf23c5f21..28437d95a5 100644 --- a/linkis-computation-governance/linkis-jdbc-driver/pom.xml +++ b/linkis-computation-governance/linkis-jdbc-driver/pom.xml @@ -41,6 +41,15 @@ + + + org.apache.maven.plugins + maven-surefire-plugin + + true + + + net.alchim31.maven scala-maven-plugin diff --git a/linkis-computation-governance/linkis-jdbc-driver/src/main/java/org/apache/linkis/ujes/jdbc/UJESSQLErrorCode.java b/linkis-computation-governance/linkis-jdbc-driver/src/main/java/org/apache/linkis/ujes/jdbc/LinkisSQLErrorCode.java similarity index 90% rename from linkis-computation-governance/linkis-jdbc-driver/src/main/java/org/apache/linkis/ujes/jdbc/UJESSQLErrorCode.java rename to linkis-computation-governance/linkis-jdbc-driver/src/main/java/org/apache/linkis/ujes/jdbc/LinkisSQLErrorCode.java index fc283d8fbe..442cbbfb43 100644 --- a/linkis-computation-governance/linkis-jdbc-driver/src/main/java/org/apache/linkis/ujes/jdbc/UJESSQLErrorCode.java +++ b/linkis-computation-governance/linkis-jdbc-driver/src/main/java/org/apache/linkis/ujes/jdbc/LinkisSQLErrorCode.java @@ -17,7 +17,7 @@ package org.apache.linkis.ujes.jdbc; -public enum UJESSQLErrorCode { +public enum LinkisSQLErrorCode { BAD_URL(80000, "bad url"), NOSUPPORT_DRIVER(80001, "this method not supported in driver"), NOSUPPORT_CONNECTION(80002, "this method not supported in connection"), @@ -38,11 +38,12 @@ public enum UJESSQLErrorCode { RESULTSET_NULL( 80017, "resultset is null,try to run next() firstly to init ResultSet and MetaData"), PREPARESTATEMENT_TYPEERROR(80018, "parameter type error"), - METADATA_EMPTY(80019, "data is empty"); + METADATA_EMPTY(80019, "data is empty"), + UNKNOWN_ERROR(80020, "unknown error"); private String msg; private int code; - UJESSQLErrorCode(int code, String msg) { + LinkisSQLErrorCode(int code, String msg) { this.code = code; this.msg = msg; } @@ -54,4 +55,8 @@ public String getMsg() { public int getCode() { return code; } + + public void setMsg(String msg) { + this.msg = msg; + } } diff --git a/linkis-computation-governance/linkis-jdbc-driver/src/main/java/org/apache/linkis/ujes/jdbc/UJESSQLDriver.java b/linkis-computation-governance/linkis-jdbc-driver/src/main/java/org/apache/linkis/ujes/jdbc/UJESSQLDriver.java index 90e1f73563..c12e2791b3 100644 --- a/linkis-computation-governance/linkis-jdbc-driver/src/main/java/org/apache/linkis/ujes/jdbc/UJESSQLDriver.java +++ b/linkis-computation-governance/linkis-jdbc-driver/src/main/java/org/apache/linkis/ujes/jdbc/UJESSQLDriver.java @@ -48,9 +48,7 @@ public class UJESSQLDriver extends UJESSQLDriverMain implements Driver { static String TOKEN_VALUE = "value"; static String PASSWORD = "password"; static boolean TABLEAU_SERVER = false; - static String LIMIT_ENABLED = "true"; - static String LIMIT = "limit"; - + static String FIXED_SESSION = "fixedSession"; static String VERSION = "version"; static int DEFAULT_VERSION = 1; static String MAX_CONNECTION_SIZE = "maxConnectionSize"; @@ -58,7 +56,7 @@ public class UJESSQLDriver extends UJESSQLDriverMain implements Driver { static String ENABLE_DISCOVERY = "enableDiscovery"; static String ENABLE_LOADBALANCER = "enableLoadBalancer"; static String CREATOR = "creator"; - + static String TABLEAU = "tableau"; static String VARIABLE_HEADER = "var:"; static String PARAM_SPLIT = "&"; static String KV_SPLIT = "="; diff --git a/linkis-computation-governance/linkis-jdbc-driver/src/main/java/org/apache/linkis/ujes/jdbc/utils/JDBCUtils.java b/linkis-computation-governance/linkis-jdbc-driver/src/main/java/org/apache/linkis/ujes/jdbc/utils/JDBCUtils.java index 061acd42f9..3e1e7e3182 100644 --- a/linkis-computation-governance/linkis-jdbc-driver/src/main/java/org/apache/linkis/ujes/jdbc/utils/JDBCUtils.java +++ b/linkis-computation-governance/linkis-jdbc-driver/src/main/java/org/apache/linkis/ujes/jdbc/utils/JDBCUtils.java @@ -17,10 +17,16 @@ package org.apache.linkis.ujes.jdbc.utils; +import org.apache.linkis.common.utils.Utils; + +import java.util.concurrent.atomic.AtomicInteger; + public class JDBCUtils { private static final char SEARCH_STRING_ESCAPE = '\\'; + public static final AtomicInteger idCreator = new AtomicInteger(); + public static String convertPattern(final String pattern) { if (pattern == null) { return ".*"; @@ -41,8 +47,6 @@ public static String convertPattern(final String pattern) { continue; } else if (c == '%') { result.append(".*"); - } else if (c == '_') { - result.append('.'); } else { result.append(Character.toLowerCase(c)); } @@ -52,4 +56,8 @@ public static String convertPattern(final String pattern) { return result.toString(); } } + + public static String getUniqId() { + return Utils.getLocalHostname() + "_" + JDBCUtils.idCreator.getAndIncrement(); + } } diff --git a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/LinkisSQLConnection.scala b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/LinkisSQLConnection.scala new file mode 100644 index 0000000000..a9b58e153b --- /dev/null +++ b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/LinkisSQLConnection.scala @@ -0,0 +1,484 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.ujes.jdbc + +import org.apache.linkis.common.utils.{Logging, Utils} +import org.apache.linkis.manager.label.constant.LabelKeyConstant +import org.apache.linkis.manager.label.entity.engine.{EngineType, EngineTypeLabel, RunType} +import org.apache.linkis.manager.label.utils.EngineTypeLabelCreator +import org.apache.linkis.ujes.client.UJESClient +import org.apache.linkis.ujes.client.request.JobSubmitAction +import org.apache.linkis.ujes.client.response.JobExecuteResult +import org.apache.linkis.ujes.jdbc.UJESSQLDriverMain._ +import org.apache.linkis.ujes.jdbc.utils.JDBCUtils + +import org.apache.commons.lang3.StringUtils + +import java.{sql, util} +import java.sql.{ + Blob, + CallableStatement, + Clob, + Connection, + DatabaseMetaData, + NClob, + PreparedStatement, + ResultSet, + Savepoint, + SQLException, + SQLWarning, + SQLXML, + Statement, + Struct +} +import java.util.Properties +import java.util.concurrent.Executor + +import scala.collection.JavaConverters._ + +class LinkisSQLConnection(private[jdbc] val ujesClient: UJESClient, props: Properties) + extends Connection + with Logging { + + private[jdbc] var creator = "JDBCDriver" + + private[jdbc] var tableauFlag = false + + private[jdbc] val variableMap = { + val params = props.getProperty(PARAMS) + val map = new util.HashMap[String, AnyRef] + if (params != null) { + params.split(PARAM_SPLIT).map(_.split(KV_SPLIT)).foreach { + case Array(k, v) if k.startsWith(VARIABLE_HEADER) => + map.put(k.substring(VARIABLE_HEADER.length), v) + case Array(CREATOR, v) => + creator = v + case _ => + } + } + map + } + + def isTableau(): Boolean = { + val params = props.getProperty(PARAMS) + if (params != null) { + params.split(PARAM_SPLIT).map(_.split(KV_SPLIT)).foreach { + case Array(TABLEAU, v) => + tableauFlag = true + case _ => + } + } + tableauFlag + } + + private[jdbc] val dbName = + if (StringUtils.isNotBlank(props.getProperty(DB_NAME))) props.getProperty(DB_NAME) + else "default" + + private val runningSQLStatements = new util.LinkedList[Statement] + + private var closed = false + + private var inited = false + + private[jdbc] val user = props.getProperty(USER) + + private[jdbc] val serverURL = props.getProperty("URL") + + private[jdbc] val fixedSessionEnabled = + if ( + props + .containsKey(FIXED_SESSION) && "true".equalsIgnoreCase(props.getProperty(FIXED_SESSION)) + ) { + true + } else { + false + } + + private val connectionId = JDBCUtils.getUniqId() + + private val labelMap: util.Map[String, AnyRef] = new util.HashMap[String, AnyRef] + + private val startupParams: util.Map[String, AnyRef] = new util.HashMap[String, AnyRef] + + private val runtimeParams: util.Map[String, AnyRef] = new util.HashMap[String, AnyRef] + + private[jdbc] def getEngineType: EngineTypeLabel = { + val engineType: EngineTypeLabel = + EngineTypeLabelCreator.createEngineTypeLabel(EngineType.TRINO.toString) + if (props.containsKey(PARAMS)) { + val params = props.getProperty(PARAMS) + if (params != null & params.length() > 0) { + params.split(PARAM_SPLIT).map(_.split(KV_SPLIT)).foreach { + case Array(k, v) if k.equals(UJESSQLDriver.ENGINE_TYPE) => + return EngineTypeLabelCreator.createEngineTypeLabel(v) + case _ => + } + } + } + engineType + } + + private[jdbc] def throwWhenClosed[T](op: => T): T = + if (isClosed) throw new LinkisSQLException(LinkisSQLErrorCode.CONNECTION_CLOSED) + else op + + private def createStatementAndAdd[T <: Statement](op: => T): T = throwWhenClosed { + + val statement = op + runningSQLStatements.add(statement) + if (!inited) { + inited = true + Utils.tryAndWarn(statement.execute(s"USE $dbName")) + } + statement + } + + def getProps: Properties = props + + def removeStatement(statement: LinkisSQLStatement): Unit = runningSQLStatements.remove(statement) + + override def createStatement(): Statement = createStatementAndAdd(new LinkisSQLStatement(this)) + + override def prepareStatement(sql: String): LinkisSQLPreparedStatement = { + val statement = createStatementAndAdd(new LinkisSQLPreparedStatement(this, sql)) + statement.clearQuery() + statement + } + + override def createStatement(resultSetType: Int, resultSetConcurrency: Int): Statement = { + if (resultSetConcurrency != ResultSet.CONCUR_READ_ONLY) { + throw new SQLException( + "Statement with resultset concurrency " + resultSetConcurrency + " is not supported", + "HYC00" + ) + } + if (resultSetType == ResultSet.TYPE_SCROLL_SENSITIVE) { + throw new SQLException( + "Statement with resultset type " + resultSetType + " is not supported", + "HYC00" + ) + } + createStatementAndAdd(new LinkisSQLStatement(this)) + } + + override def prepareStatement(sql: String, autoGeneratedKeys: Int): PreparedStatement = + prepareStatement(sql) + + override def prepareStatement( + sql: String, + resultSetType: Int, + resultSetConcurrency: Int + ): PreparedStatement = prepareStatement(sql) + + override def getMetaData: DatabaseMetaData = throwWhenClosed(new UJESSQLDatabaseMetaData(this)) + + override def close(): Unit = { + runningSQLStatements.asScala.foreach { statement => Utils.tryQuietly(statement.close()) } + closed = true + } + + override def isClosed: Boolean = closed + + override def setReadOnly(readOnly: Boolean): Unit = + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "setReadOnly not supported" + ) + + override def isReadOnly: Boolean = false + + override def setCatalog(catalog: String): Unit = throwWhenClosed() + + override def getCatalog: String = "" + + override def setTransactionIsolation(level: Int): Unit = {} + + override def getTransactionIsolation: Int = Connection.TRANSACTION_NONE + + override def getWarnings: SQLWarning = null + + override def clearWarnings(): Unit = {} + + override def setAutoCommit(autoCommit: Boolean): Unit = {} + + override def getAutoCommit: Boolean = true + + override def commit(): Unit = {} + + override def prepareCall(sql: String): CallableStatement = + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "prepareCall not supported" + ) + + override def rollback(): Unit = + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_CONNECTION, "rollback not supported") + + override def nativeSQL(sql: String): String = + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_CONNECTION, "nativeSQL not supported") + + override def prepareCall( + sql: String, + resultSetType: Int, + resultSetConcurrency: Int + ): CallableStatement = + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "prepareCall not supported" + ) + + override def getTypeMap: util.Map[String, Class[_]] = + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "getTypeMap not supported" + ) + + override def setTypeMap(map: util.Map[String, Class[_]]): Unit = + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "setTypeMap not supported" + ) + + override def setHoldability(holdability: Int): Unit = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "setHoldability not supported" + ) + + override def getHoldability: Int = 0 + + override def setSavepoint(): Savepoint = + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "setSavepoint not supported" + ) + + override def setSavepoint(name: String): Savepoint = + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "setSavepoint not supported" + ) + + override def rollback(savepoint: Savepoint): Unit = + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_CONNECTION, "rollback not supported") + + override def releaseSavepoint(savepoint: Savepoint): Unit = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "releaseSavepoint not supported" + ) + + override def createStatement( + resultSetType: Int, + resultSetConcurrency: Int, + resultSetHoldability: Int + ): Statement = + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "createStatement not supported" + ) + + override def prepareStatement( + sql: String, + resultSetType: Int, + resultSetConcurrency: Int, + resultSetHoldability: Int + ): PreparedStatement = + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "prepareStatement not supported" + ) + + override def prepareCall( + sql: String, + resultSetType: Int, + resultSetConcurrency: Int, + resultSetHoldability: Int + ): CallableStatement = + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "prepareCall not supported" + ) + + override def prepareStatement(sql: String, columnIndexes: Array[Int]): PreparedStatement = + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "prepareStatement not supported" + ) + + override def prepareStatement(sql: String, columnNames: Array[String]): PreparedStatement = + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "prepareStatement not supported" + ) + + override def createClob(): Clob = + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "createClob not supported" + ) + + override def createBlob(): Blob = + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "createBlob not supported" + ) + + override def createNClob(): NClob = + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "createNClob not supported" + ) + + override def createSQLXML(): SQLXML = + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "createSQLXML not supported" + ) + + override def isValid(timeout: Int): Boolean = true + + override def setClientInfo(name: String, value: String): Unit = + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "setClientInfo not supported" + ) + + override def setClientInfo(properties: Properties): Unit = + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "properties not supported" + ) + + override def getClientInfo(name: String): String = + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "getClientInfo not supported" + ) + + override def getClientInfo: Properties = + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "getClientInfo not supported" + ) + + override def createArrayOf(typeName: String, elements: Array[AnyRef]): sql.Array = + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "createArrayOf not supported" + ) + + override def createStruct(typeName: String, attributes: Array[AnyRef]): Struct = + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "createStruct not supported" + ) + + override def setSchema(schema: String): Unit = throwWhenClosed { + if (StringUtils.isBlank(schema)) { + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_STATEMENT, "schema is empty!") + } + createStatement().execute("use " + schema) + } + + override def getSchema: String = throwWhenClosed { + val resultSet = createStatement().executeQuery("SELECT current_database()") + if (!resultSet.next()) { + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_STATEMENT, "Get schema failed!") + } + resultSet.getString(1) + } + + override def abort(executor: Executor): Unit = + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_CONNECTION, "abort not supported") + + override def setNetworkTimeout(executor: Executor, milliseconds: Int): Unit = + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "setNetworkTimeout not supported" + ) + + override def getNetworkTimeout: Int = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "getNetworkTimeout not supported" + ) + + override def unwrap[T](iface: Class[T]): T = + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_CONNECTION, "unwrap not supported") + + override def isWrapperFor(iface: Class[_]): Boolean = + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "isWrapperFor not supported" + ) + + def addLabels(labels: util.Map[String, AnyRef]): Unit = { + labelMap.putAll(labels) + } + + def addStartUpParams(params: util.Map[String, AnyRef]): Unit = { + startupParams.putAll(params) + } + + def addRuntimeParams(params: util.Map[String, AnyRef]): Unit = { + runtimeParams.putAll(params) + } + + def engineToCodeType(engine: String): String = { + val runType = EngineType.mapStringToEngineType(engine) match { + case EngineType.SPARK => RunType.SQL + case EngineType.HIVE => RunType.HIVE + case EngineType.TRINO => RunType.TRINO_SQL + case EngineType.PRESTO => RunType.PRESTO_SQL + case EngineType.ELASTICSEARCH => RunType.ES_SQL + case EngineType.JDBC => RunType.JDBC + case EngineType.PYTHON => RunType.SHELL + case _ => RunType.SQL + } + runType.toString + } + + private[jdbc] def toSubmit(code: String): JobExecuteResult = { + val engineTypeLabel = getEngineType + labelMap.put(LabelKeyConstant.ENGINE_TYPE_KEY, engineTypeLabel.getStringValue) + labelMap.put(LabelKeyConstant.USER_CREATOR_TYPE_KEY, s"$user-$creator") + labelMap.put(LabelKeyConstant.CODE_TYPE_KEY, engineToCodeType(engineTypeLabel.getEngineType)) + if (fixedSessionEnabled) { + labelMap.put(LabelKeyConstant.FIXED_EC_KEY, connectionId) + logger.info("Fixed session is enable session id is {}", connectionId) + } + + val jobSubmitAction = JobSubmitAction.builder + .addExecuteCode(code) + .setStartupParams(startupParams) + .setUser(user) + .addExecuteUser(user) + .setLabels(labelMap) + .setRuntimeParams(runtimeParams) + .setVariableMap(variableMap) + .build + + val result = ujesClient.submit(jobSubmitAction) + if (result.getStatus != 0) { + throw new SQLException(result.getMessage) + } + result + } + + override def toString: String = "LinkisConnection_" + connectionId + +} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/data/PreparedData.java b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/LinkisSQLException.scala similarity index 64% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/data/PreparedData.java rename to linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/LinkisSQLException.scala index bdde39e23e..9cc3f3814f 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/data/PreparedData.java +++ b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/LinkisSQLException.scala @@ -15,20 +15,23 @@ * limitations under the License. */ -package org.apache.linkis.cli.application.data; +package org.apache.linkis.ujes.jdbc -import org.apache.linkis.cli.common.entity.properties.ClientProperties; +import java.sql.SQLException -import java.util.Map; +class LinkisSQLException(msg: String, code: String, vendorCode: Int) + extends SQLException(msg, code, vendorCode) { -public class PreparedData { - Map propertiesMap; + def this(errorCode: LinkisSQLErrorCode, msg: String) { + this(msg, errorCode.getCode.toString, 0) + } - public PreparedData(Map propertiesMap) { - this.propertiesMap = propertiesMap; + def this(errorCode: LinkisSQLErrorCode) { + this(errorCode.getMsg, errorCode.getCode.toString, 0) } - public Map getPropertiesMap() { - return propertiesMap; + def this(msg: String, code: String) { + this(msg, code, 0) } + } diff --git a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLPreparedStatement.scala b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/LinkisSQLPreparedStatement.scala similarity index 77% rename from linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLPreparedStatement.scala rename to linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/LinkisSQLPreparedStatement.scala index 6328da99e2..61a7020946 100644 --- a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLPreparedStatement.scala +++ b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/LinkisSQLPreparedStatement.scala @@ -36,8 +36,8 @@ import java.sql.{ import java.util import java.util.Calendar -class UJESSQLPreparedStatement(ujesSQLConnection: UJESSQLConnection, sql: String) - extends UJESSQLStatement(ujesSQLConnection) +class LinkisSQLPreparedStatement(ujesSQLConnection: LinkisSQLConnection, sql: String) + extends LinkisSQLStatement(ujesSQLConnection) with PreparedStatement { private val parameters = new util.HashMap[Int, Any] @@ -139,7 +139,7 @@ class UJESSQLPreparedStatement(ujesSQLConnection: UJESSQLConnection, sql: String } override def setTime(parameterIndex: Int, x: Time): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_STATEMENT) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_STATEMENT) } override def setTimestamp(parameterIndex: Int, x: Timestamp): Unit = { @@ -147,15 +147,15 @@ class UJESSQLPreparedStatement(ujesSQLConnection: UJESSQLConnection, sql: String } override def setAsciiStream(parameterIndex: Int, x: InputStream, length: Int): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_STATEMENT) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_STATEMENT) } override def setUnicodeStream(parameterIndex: Int, x: InputStream, length: Int): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_STATEMENT) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_STATEMENT) } override def setBinaryStream(parameterIndex: Int, x: InputStream, length: Int): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_STATEMENT) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_STATEMENT) } override def clearParameters(): Unit = { @@ -163,7 +163,7 @@ class UJESSQLPreparedStatement(ujesSQLConnection: UJESSQLConnection, sql: String } override def setObject(parameterIndex: Int, x: scala.Any, targetSqlType: Int): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_STATEMENT) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_STATEMENT) } override def setObject(parameterIndex: Int, x: scala.Any): Unit = { @@ -180,8 +180,8 @@ class UJESSQLPreparedStatement(ujesSQLConnection: UJESSQLConnection, sql: String case x: Char => setString(parameterIndex, x.toString) case x: Timestamp => setTimestamp(parameterIndex, x) case _ => - throw new UJESSQLException( - UJESSQLErrorCode.PREPARESTATEMENT_TYPEERROR, + throw new LinkisSQLException( + LinkisSQLErrorCode.PREPARESTATEMENT_TYPEERROR, s"Can''t infer the SQL type to use for an instance of ${x.getClass.getName}. Use setObject() with an explicit Types value to specify the type to use" ) } @@ -208,23 +208,23 @@ class UJESSQLPreparedStatement(ujesSQLConnection: UJESSQLConnection, sql: String } override def setCharacterStream(parameterIndex: Int, reader: Reader, length: Int): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_STATEMENT) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_STATEMENT) } override def setRef(parameterIndex: Int, x: Ref): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_STATEMENT) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_STATEMENT) } override def setBlob(parameterIndex: Int, x: Blob): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_STATEMENT) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_STATEMENT) } override def setClob(parameterIndex: Int, x: Clob): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_STATEMENT) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_STATEMENT) } override def setArray(parameterIndex: Int, x: java.sql.Array): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_STATEMENT) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_STATEMENT) } override def getMetaData: ResultSetMetaData = { @@ -235,15 +235,15 @@ class UJESSQLPreparedStatement(ujesSQLConnection: UJESSQLConnection, sql: String } override def setDate(parameterIndex: Int, x: Date, cal: Calendar): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_STATEMENT) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_STATEMENT) } override def setTime(parameterIndex: Int, x: Time, cal: Calendar): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_STATEMENT) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_STATEMENT) } override def setTimestamp(parameterIndex: Int, x: Timestamp, cal: Calendar): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_STATEMENT) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_STATEMENT) } override def setNull(parameterIndex: Int, sqlType: Int, typeName: String): Unit = { @@ -251,7 +251,7 @@ class UJESSQLPreparedStatement(ujesSQLConnection: UJESSQLConnection, sql: String } override def setURL(parameterIndex: Int, x: URL): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_STATEMENT) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_STATEMENT) } override def getParameterMetaData: ParameterMetaData = { @@ -264,35 +264,35 @@ class UJESSQLPreparedStatement(ujesSQLConnection: UJESSQLConnection, sql: String } override def setRowId(parameterIndex: Int, x: RowId): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_STATEMENT) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_STATEMENT) } override def setNString(parameterIndex: Int, value: String): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_STATEMENT) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_STATEMENT) } override def setNCharacterStream(parameterIndex: Int, value: Reader, length: Long): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_STATEMENT) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_STATEMENT) } override def setNClob(parameterIndex: Int, value: NClob): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_STATEMENT) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_STATEMENT) } override def setClob(parameterIndex: Int, reader: Reader, length: Long): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_STATEMENT) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_STATEMENT) } override def setBlob(parameterIndex: Int, inputStream: InputStream, length: Long): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_STATEMENT) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_STATEMENT) } override def setNClob(parameterIndex: Int, reader: Reader, length: Long): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_STATEMENT) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_STATEMENT) } override def setSQLXML(parameterIndex: Int, xmlObject: SQLXML): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_STATEMENT) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_STATEMENT) } override def setObject( @@ -301,47 +301,47 @@ class UJESSQLPreparedStatement(ujesSQLConnection: UJESSQLConnection, sql: String targetSqlType: Int, scaleOrLength: Int ): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_STATEMENT) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_STATEMENT) } override def setAsciiStream(parameterIndex: Int, x: InputStream, length: Long): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_STATEMENT) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_STATEMENT) } override def setBinaryStream(parameterIndex: Int, x: InputStream, length: Long): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_STATEMENT) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_STATEMENT) } override def setCharacterStream(parameterIndex: Int, reader: Reader, length: Long): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_STATEMENT) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_STATEMENT) } override def setAsciiStream(parameterIndex: Int, x: InputStream): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_STATEMENT) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_STATEMENT) } override def setBinaryStream(parameterIndex: Int, x: InputStream): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_STATEMENT) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_STATEMENT) } override def setCharacterStream(parameterIndex: Int, reader: Reader): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_STATEMENT) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_STATEMENT) } override def setNCharacterStream(parameterIndex: Int, value: Reader): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_STATEMENT) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_STATEMENT) } override def setClob(parameterIndex: Int, reader: Reader): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_STATEMENT) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_STATEMENT) } override def setBlob(parameterIndex: Int, inputStream: InputStream): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_STATEMENT) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_STATEMENT) } override def setNClob(parameterIndex: Int, reader: Reader): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_STATEMENT) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_STATEMENT) } override def getResultSetType: Int = { diff --git a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLStatement.scala b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/LinkisSQLStatement.scala similarity index 50% rename from linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLStatement.scala rename to linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/LinkisSQLStatement.scala index 916bdbaa93..f00d870978 100644 --- a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLStatement.scala +++ b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/LinkisSQLStatement.scala @@ -17,21 +17,22 @@ package org.apache.linkis.ujes.jdbc -import org.apache.linkis.common.exception.ErrorException import org.apache.linkis.common.utils.{Logging, Utils} -import org.apache.linkis.ujes.client.request.JobExecuteAction -import org.apache.linkis.ujes.client.request.JobExecuteAction.EngineType +import org.apache.linkis.governance.common.entity.ExecutionNodeStatus +import org.apache.linkis.ujes.client.request.OpenLogAction import org.apache.linkis.ujes.client.response.JobExecuteResult import org.apache.linkis.ujes.jdbc.hook.JDBCDriverPreExecutionHook -import java.sql.{Connection, ResultSet, SQLWarning, Statement} +import org.apache.commons.lang3.StringUtils + +import java.sql.{Connection, ResultSet, SQLException, SQLWarning, Statement} +import java.util import java.util.concurrent.TimeUnit -import scala.collection.JavaConverters.mapAsJavaMapConverter import scala.concurrent.TimeoutException import scala.concurrent.duration.Duration -class UJESSQLStatement(private[jdbc] val ujesSQLConnection: UJESSQLConnection) +class LinkisSQLStatement(private[jdbc] val ujesSQLConnection: LinkisSQLConnection) extends Statement with Logging { @@ -42,15 +43,20 @@ class UJESSQLStatement(private[jdbc] val ujesSQLConnection: UJESSQLConnection) private var fetchSize = 100 private var queryTimeout = 0 + private var logPath: String = null + private var queryEnd = false + private var logFromLen = 0 + private val logSize = 100 + private[jdbc] def throwWhenClosed[T](op: => T): T = ujesSQLConnection.throwWhenClosed { - if (isClosed) throw new UJESSQLException(UJESSQLErrorCode.STATEMENT_CLOSED) + if (isClosed) throw new LinkisSQLException(LinkisSQLErrorCode.STATEMENT_CLOSED) else op } override def executeQuery(sql: String): UJESSQLResultSet = { - if (!execute(sql)) throw new UJESSQLException(UJESSQLErrorCode.RESULTSET_NULL) + if (!execute(sql)) throw new LinkisSQLException(LinkisSQLErrorCode.RESULTSET_NULL) resultSet } @@ -75,13 +81,13 @@ class UJESSQLStatement(private[jdbc] val ujesSQLConnection: UJESSQLConnection) } } - override def getMaxFieldSize: Int = throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_STATEMENT, + override def getMaxFieldSize: Int = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_STATEMENT, "getMaxFieldSize not supported" ) - override def setMaxFieldSize(max: Int): Unit = throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_STATEMENT, + override def setMaxFieldSize(max: Int): Unit = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_STATEMENT, "setMaxFieldSize not supported" ) @@ -89,11 +95,12 @@ class UJESSQLStatement(private[jdbc] val ujesSQLConnection: UJESSQLConnection) override def setMaxRows(max: Int): Unit = this.maxRows = max - override def setEscapeProcessing(enable: Boolean): Unit = if (enable) - throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_STATEMENT, + override def setEscapeProcessing(enable: Boolean): Unit = if (enable) { + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_STATEMENT, "setEscapeProcessing not supported" ) + } override def getQueryTimeout: Int = queryTimeout @@ -106,90 +113,75 @@ class UJESSQLStatement(private[jdbc] val ujesSQLConnection: UJESSQLConnection) override def clearWarnings(): Unit = {} override def setCursorName(name: String): Unit = - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_STATEMENT, "setCursorName not supported") + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_STATEMENT, + "setCursorName not supported" + ) - override def execute(sql: String): Boolean = Utils.tryCatch(throwWhenClosed { + override def execute(sql: String): Boolean = throwWhenClosed { var parsedSQL = sql JDBCDriverPreExecutionHook.getPreExecutionHooks.foreach { preExecution => - parsedSQL = preExecution.callPreExecutionHook(parsedSQL) + parsedSQL = preExecution.callPreExecutionHook(parsedSQL, !ujesSQLConnection.isTableau()) } logger.info(s"begin to execute sql ${parsedSQL}") - val action = JobExecuteAction - .builder() - .setEngineType(ujesSQLConnection.getEngineType) - .addExecuteCode(parsedSQL) - .setCreator(ujesSQLConnection.creator) - .setUser(ujesSQLConnection.user) - if (ujesSQLConnection.variableMap.nonEmpty) { - action.setVariableMap(ujesSQLConnection.variableMap.asJava) - } - jobExecuteResult = - Utils.tryCatch(ujesSQLConnection.ujesClient.execute(action.build())) { t: Throwable => - logger.error("UJESClient failed to get result", t) - null - } - // jobExecuteResult = ujesSQLConnection.ujesClient.execute(action.build()) queryEnd = false - var status = ujesSQLConnection.ujesClient.status(jobExecuteResult) - val atMost = - if (queryTimeout > 0) Duration(queryTimeout, TimeUnit.MILLISECONDS) else Duration.Inf - if (!status.isCompleted) Utils.tryThrow { - Utils.waitUntil( - () => { - status = ujesSQLConnection.ujesClient.status(jobExecuteResult) - status.isCompleted || closed - }, - atMost, - 100, - 10000 - ) - } { - case t: TimeoutException => - if (queryTimeout > 0) clearQuery() - new UJESSQLException(UJESSQLErrorCode.QUERY_TIMEOUT, "query has been timeout!").initCause(t) - case t => t - } - if (!closed) { + logPath = null + try { + jobExecuteResult = ujesSQLConnection.toSubmit(parsedSQL) + val atMost = + if (queryTimeout > 0) Duration(queryTimeout, TimeUnit.MILLISECONDS) else Duration.Inf var jobInfo = ujesSQLConnection.ujesClient.getJobInfo(jobExecuteResult) - if (status.isFailed) - throw new ErrorException( - jobInfo.getRequestPersistTask.getErrCode, - jobInfo.getRequestPersistTask.getErrDesc - ) - val jobInfoStatus = jobInfo.getJobStatus - if (!jobInfoStatus.equals("Succeed")) Utils.tryThrow { - Utils.waitUntil( - () => { - jobInfo = ujesSQLConnection.ujesClient.getJobInfo(jobExecuteResult) - val state = jobInfo.getJobStatus match { - case "Failed" | "Cancelled" | "Timeout" | "Succeed" => true - case _ => false - } - state || closed - }, - atMost, - 100, - 10000 + logPath = jobInfo.getRequestPersistTask.getLogPath + if (!ExecutionNodeStatus.isCompleted(ExecutionNodeStatus.valueOf(jobInfo.getJobStatus))) { + Utils.tryThrow { + Utils.waitUntil( + () => { + jobInfo = ujesSQLConnection.ujesClient.getJobInfo(jobExecuteResult) + ExecutionNodeStatus.isCompleted( + ExecutionNodeStatus.valueOf(jobInfo.getJobStatus) + ) || closed + }, + atMost, + 100, + 10000 + ) + } { + case t: TimeoutException => + if (queryTimeout > 0) clearQuery() + logPath = jobInfo.getRequestPersistTask.getLogPath + new LinkisSQLException(LinkisSQLErrorCode.QUERY_TIMEOUT, "query has been timeout!") + .initCause(t) + case t => t + } + } + logPath = jobInfo.getRequestPersistTask.getLogPath + if (!ExecutionNodeStatus.isSucceed(ExecutionNodeStatus.valueOf(jobInfo.getJobStatus))) { + throw new LinkisSQLException( + jobInfo.getRequestPersistTask.getErrDesc, + jobInfo.getRequestPersistTask.getErrCode.toString ) - } { - case t: TimeoutException => - if (queryTimeout > 0) clearQuery() - new UJESSQLException(UJESSQLErrorCode.QUERY_TIMEOUT, "query has been timeout!") - .initCause(t) - case t => t } + logger.info(s"end to execute sql ${parsedSQL}") val resultSetList = jobInfo.getResultSetList(ujesSQLConnection.ujesClient) logger.info(s"resultSetList is ${resultSetList.mkString(",")}") - queryEnd = true - if (resultSetList != null) { + if (resultSetList != null && resultSetList.nonEmpty) { resultSet = new UJESSQLResultSet(resultSetList, this, maxRows, fetchSize) true - } else false - } else throw new UJESSQLException(UJESSQLErrorCode.STATEMENT_CLOSED, "Statement is closed.") - }) { case t: Throwable => - logger.error("任务执行失败", t) - false + } else { + false + } + } catch { + case sqlException: SQLException => + throw sqlException + case throwable: Throwable => + val exception = + new LinkisSQLException(LinkisSQLErrorCode.UNKNOWN_ERROR, throwable.getMessage) + exception.initCause(throwable) + throw exception + } finally { + queryEnd = true + } } def getJobExcuteResult: JobExecuteResult = jobExecuteResult @@ -201,13 +193,12 @@ class UJESSQLStatement(private[jdbc] val ujesSQLConnection: UJESSQLConnection) override def getMoreResults: Boolean = false override def setFetchDirection(direction: Int): Unit = - throwWhenClosed( - if (direction != ResultSet.FETCH_FORWARD) - throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_STATEMENT, - "only FETCH_FORWARD is supported." - ) - ) + throwWhenClosed(if (direction != ResultSet.FETCH_FORWARD) { + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_STATEMENT, + "only FETCH_FORWARD is supported." + ) + }) override def getFetchDirection: Int = throwWhenClosed(ResultSet.FETCH_FORWARD) @@ -215,88 +206,126 @@ class UJESSQLStatement(private[jdbc] val ujesSQLConnection: UJESSQLConnection) override def getFetchSize: Int = fetchSize - override def getResultSetConcurrency: Int = throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_STATEMENT, + override def getResultSetConcurrency: Int = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_STATEMENT, "getResultSetConcurrency not supported." ) override def getResultSetType: Int = throwWhenClosed(ResultSet.TYPE_FORWARD_ONLY) override def addBatch(sql: String): Unit = - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_STATEMENT, "addBatch not supported.") + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_STATEMENT, "addBatch not supported.") override def clearBatch(): Unit = - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_STATEMENT, "clearBatch not supported.") + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_STATEMENT, + "clearBatch not supported." + ) override def executeBatch(): Array[Int] = - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_STATEMENT, "executeBatch not supported.") + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_STATEMENT, + "executeBatch not supported." + ) override def getConnection: Connection = throwWhenClosed(ujesSQLConnection) override def getMoreResults(current: Int): Boolean = false - override def getGeneratedKeys: ResultSet = throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_STATEMENT, + override def getGeneratedKeys: ResultSet = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_STATEMENT, "getGeneratedKeys not supported." ) override def executeUpdate(sql: String, autoGeneratedKeys: Int): Int = - throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_STATEMENT, + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_STATEMENT, "executeUpdate with autoGeneratedKeys not supported." ) override def executeUpdate(sql: String, columnIndexes: Array[Int]): Int = - throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_STATEMENT, + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_STATEMENT, "executeUpdate with columnIndexes not supported." ) override def executeUpdate(sql: String, columnNames: Array[String]): Int = - throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_STATEMENT, + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_STATEMENT, "executeUpdate with columnNames not supported." ) override def execute(sql: String, autoGeneratedKeys: Int): Boolean = - throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_STATEMENT, + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_STATEMENT, "execute with autoGeneratedKeys not supported." ) override def execute(sql: String, columnIndexes: Array[Int]): Boolean = - throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_STATEMENT, + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_STATEMENT, "execute with columnIndexes not supported." ) override def execute(sql: String, columnNames: Array[String]): Boolean = - throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_STATEMENT, + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_STATEMENT, "execute with columnNames not supported." ) - override def getResultSetHoldability: Int = throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_STATEMENT, + override def getResultSetHoldability: Int = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_STATEMENT, "getResultSetHoldability not supported" ) override def isClosed: Boolean = closed override def setPoolable(poolable: Boolean): Unit = - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_STATEMENT, "setPoolable not supported") + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_STATEMENT, + "setPoolable not supported" + ) override def isPoolable: Boolean = false - override def closeOnCompletion(): Unit = throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_STATEMENT, + override def closeOnCompletion(): Unit = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_STATEMENT, "closeOnCompletion not supported" ) override def isCloseOnCompletion: Boolean = false override def unwrap[T](iface: Class[T]): T = - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_STATEMENT, "unwrap not supported") + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_STATEMENT, "unwrap not supported") override def isWrapperFor(iface: Class[_]): Boolean = false + + /** + * log[0] error log[1] warn log[2] info log[3] all (info + warn + error) + * @return + */ + def getAllLog(): Array[String] = { + if (queryEnd && StringUtils.isNotBlank(logPath)) { + val openLogAction = + OpenLogAction.newBuilder().setLogPath(logPath).setProxyUser(ujesSQLConnection.user).build() + ujesSQLConnection.ujesClient.openLog(openLogAction).getLog + } else { + Array.empty[String] + } + } + + /** + * log[0] error log[1] warn log[2] info log[3] all (info + warn + error) + * @return + */ + def getIncrementalLog(): util.List[String] = { + if (null != jobExecuteResult && !queryEnd) { + val logObj = ujesSQLConnection.ujesClient.log(jobExecuteResult, logFromLen, logSize) + logFromLen = logObj.fromLine + logObj.getLog + } else { + new util.ArrayList[String] + } + } + } diff --git a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESClientFactory.scala b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESClientFactory.scala index 60f4d88af5..bea24181a3 100644 --- a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESClientFactory.scala +++ b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESClientFactory.scala @@ -35,15 +35,27 @@ object UJESClientFactory extends Logging { def getUJESClient(props: Properties): UJESClient = { val host = props.getProperty(HOST) val port = props.getProperty(PORT) + val user = props.getProperty(USER) val serverUrl = if (StringUtils.isNotBlank(port)) s"http://$host:$port" else "http://" + host - if (ujesClients.containsKey(serverUrl)) ujesClients.get(serverUrl) - else - serverUrl.intern synchronized { - if (ujesClients.containsKey(serverUrl)) return ujesClients.get(serverUrl) + val uniqueKey = s"${serverUrl}_$user" + if (ujesClients.containsKey(uniqueKey)) { + logger.info("Clients with the same JDBC unique key({}) will get it directly", uniqueKey) + ujesClients.get(uniqueKey) + } else { + uniqueKey.intern synchronized { + if (ujesClients.containsKey(uniqueKey)) { + logger.info("Clients with the same JDBC unique key({}) will get it directly", uniqueKey) + return ujesClients.get(uniqueKey) + } + logger.info( + "The same Client does not exist for the JDBC unique key({}), a new Client will be created", + uniqueKey + ) val ujesClient = createUJESClient(serverUrl, props) - ujesClients.put(serverUrl, ujesClient) + ujesClients.put(uniqueKey, ujesClient) ujesClient } + } } private def createUJESClient(serverUrl: String, props: Properties): UJESClient = { @@ -54,7 +66,6 @@ object UJESClientFactory extends Logging { clientConfigBuilder.setAuthenticationStrategy(new StaticAuthenticationStrategy()) clientConfigBuilder.readTimeout(100000) clientConfigBuilder.maxConnectionSize(20) - clientConfigBuilder.readTimeout(10000) val params = props.getProperty(PARAMS) var versioned = false if (StringUtils.isNotBlank(params)) { diff --git a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLConnection.scala b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLConnection.scala deleted file mode 100644 index 0d8403c274..0000000000 --- a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLConnection.scala +++ /dev/null @@ -1,340 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.ujes.jdbc - -import org.apache.linkis.common.utils.{Logging, Utils} -import org.apache.linkis.ujes.client.UJESClient -import org.apache.linkis.ujes.client.request.JobExecuteAction.EngineType -import org.apache.linkis.ujes.jdbc.UJESSQLDriverMain._ - -import org.apache.commons.lang3.StringUtils - -import java.{sql, util} -import java.sql.{ - Blob, - CallableStatement, - Clob, - Connection, - DatabaseMetaData, - NClob, - PreparedStatement, - ResultSet, - Savepoint, - SQLException, - SQLWarning, - SQLXML, - Statement, - Struct -} -import java.util.Properties -import java.util.concurrent.Executor - -import scala.collection.{mutable, JavaConversions} - -class UJESSQLConnection(private[jdbc] val ujesClient: UJESClient, props: Properties) - extends Connection - with Logging { - private[jdbc] var creator = "IDE" - - private[jdbc] val variableMap = { - val params = props.getProperty(PARAMS) - val map = new mutable.HashMap[String, AnyRef] - if (params != null) { - params.split(PARAM_SPLIT).map(_.split(KV_SPLIT)).foreach { - case Array(k, v) if k.startsWith(VARIABLE_HEADER) => - map += k.substring(VARIABLE_HEADER.length) -> v - case Array(CREATOR, v) => - creator = v - case _ => - } - } - map.toMap - } - - private[jdbc] val dbName = - if (StringUtils.isNotBlank(props.getProperty(DB_NAME))) props.getProperty(DB_NAME) - else "default" - - private val runningSQLStatements = new util.LinkedList[Statement] - - private var closed = false - - private var inited = false - - private[jdbc] val user = props.getProperty(USER) - - private[jdbc] val serverURL = props.getProperty("URL") - - private val engineTypeMap: mutable.HashMap[String, EngineType] = new mutable.HashMap() - - private[jdbc] def getEngineType: EngineType = { - if (engineTypeMap.isEmpty) { - engineTypeMap.put(EngineType.SPARK.toString, EngineType.SPARK) - engineTypeMap.put(EngineType.HIVE.toString, EngineType.HIVE) - engineTypeMap.put(EngineType.JDBC.toString, EngineType.JDBC) - engineTypeMap.put(EngineType.PYTHON.toString, EngineType.PYTHON) - engineTypeMap.put(EngineType.SHELL.toString, EngineType.SHELL) - engineTypeMap.put(EngineType.PRESTO.toString, EngineType.PRESTO) - } - val engineType: EngineType = EngineType.PRESTO - if (props.containsKey(PARAMS)) { - val params = props.getProperty(PARAMS) - if (params != null & params.length() > 0) { - params.split(PARAM_SPLIT).map(_.split(KV_SPLIT)).foreach { - case Array(k, v) if k.equals(UJESSQLDriver.ENGINE_TYPE) => return engineTypeMap(v) - case _ => - } - } - } - engineType - } - - private[jdbc] def throwWhenClosed[T](op: => T): T = - if (isClosed) throw new UJESSQLException(UJESSQLErrorCode.CONNECTION_CLOSED) - else op - - private def createStatementAndAdd[T <: Statement](op: => T): T = throwWhenClosed { - - val statement = op - runningSQLStatements.add(statement) - if (!inited) { - inited = true - Utils.tryAndWarn(statement.execute(s"USE $dbName")) - } - statement - } - - def getProps: Properties = props - - def removeStatement(statement: UJESSQLStatement): Unit = runningSQLStatements.remove(statement) - - override def createStatement(): Statement = createStatementAndAdd(new UJESSQLStatement(this)) - - override def prepareStatement(sql: String): UJESSQLPreparedStatement = { - val statement = createStatementAndAdd(new UJESSQLPreparedStatement(this, sql)) - statement.clearQuery() - statement - } - - override def createStatement(resultSetType: Int, resultSetConcurrency: Int): Statement = { - if (resultSetConcurrency != ResultSet.CONCUR_READ_ONLY) - throw new SQLException( - "Statement with resultset concurrency " + resultSetConcurrency + " is not supported", - "HYC00" - ) - if (resultSetType == ResultSet.TYPE_SCROLL_SENSITIVE) - throw new SQLException( - "Statement with resultset type " + resultSetType + " is not supported", - "HYC00" - ) - createStatementAndAdd(new UJESSQLStatement(this)) - } - - override def prepareStatement(sql: String, autoGeneratedKeys: Int): PreparedStatement = - prepareStatement(sql) - - override def prepareStatement( - sql: String, - resultSetType: Int, - resultSetConcurrency: Int - ): PreparedStatement = prepareStatement(sql) - - override def getMetaData: DatabaseMetaData = throwWhenClosed(new UJESSQLDatabaseMetaData(this)) - - override def close(): Unit = { - JavaConversions - .asScalaBuffer(runningSQLStatements) - .foreach(statement => Utils.tryQuietly(statement.close())) - closed = true - } - - override def isClosed: Boolean = closed - - override def setReadOnly(readOnly: Boolean): Unit = - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_CONNECTION, "setReadOnly not supported") - - override def isReadOnly: Boolean = false - - override def setCatalog(catalog: String): Unit = throwWhenClosed() - - override def getCatalog: String = "" - - override def setTransactionIsolation(level: Int): Unit = {} - - override def getTransactionIsolation: Int = Connection.TRANSACTION_NONE - - override def getWarnings: SQLWarning = null - - override def clearWarnings(): Unit = {} - - override def setAutoCommit(autoCommit: Boolean): Unit = {} - - override def getAutoCommit: Boolean = true - - override def commit(): Unit = {} - - override def prepareCall(sql: String): CallableStatement = - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_CONNECTION, "prepareCall not supported") - - override def rollback(): Unit = - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_CONNECTION, "rollback not supported") - - override def nativeSQL(sql: String): String = - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_CONNECTION, "nativeSQL not supported") - - override def prepareCall( - sql: String, - resultSetType: Int, - resultSetConcurrency: Int - ): CallableStatement = - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_CONNECTION, "prepareCall not supported") - - override def getTypeMap: util.Map[String, Class[_]] = - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_CONNECTION, "getTypeMap not supported") - - override def setTypeMap(map: util.Map[String, Class[_]]): Unit = - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_CONNECTION, "setTypeMap not supported") - - override def setHoldability(holdability: Int): Unit = throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_CONNECTION, - "setHoldability not supported" - ) - - override def getHoldability: Int = 0 - - override def setSavepoint(): Savepoint = - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_CONNECTION, "setSavepoint not supported") - - override def setSavepoint(name: String): Savepoint = - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_CONNECTION, "setSavepoint not supported") - - override def rollback(savepoint: Savepoint): Unit = - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_CONNECTION, "rollback not supported") - - override def releaseSavepoint(savepoint: Savepoint): Unit = throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_CONNECTION, - "releaseSavepoint not supported" - ) - - override def createStatement( - resultSetType: Int, - resultSetConcurrency: Int, - resultSetHoldability: Int - ): Statement = - throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_CONNECTION, - "createStatement not supported" - ) - - override def prepareStatement( - sql: String, - resultSetType: Int, - resultSetConcurrency: Int, - resultSetHoldability: Int - ): PreparedStatement = - throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_CONNECTION, - "prepareStatement not supported" - ) - - override def prepareCall( - sql: String, - resultSetType: Int, - resultSetConcurrency: Int, - resultSetHoldability: Int - ): CallableStatement = - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_CONNECTION, "prepareCall not supported") - - override def prepareStatement(sql: String, columnIndexes: Array[Int]): PreparedStatement = - throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_CONNECTION, - "prepareStatement not supported" - ) - - override def prepareStatement(sql: String, columnNames: Array[String]): PreparedStatement = - throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_CONNECTION, - "prepareStatement not supported" - ) - - override def createClob(): Clob = - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_CONNECTION, "createClob not supported") - - override def createBlob(): Blob = - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_CONNECTION, "createBlob not supported") - - override def createNClob(): NClob = - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_CONNECTION, "createNClob not supported") - - override def createSQLXML(): SQLXML = - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_CONNECTION, "createSQLXML not supported") - - override def isValid(timeout: Int): Boolean = true - - override def setClientInfo(name: String, value: String): Unit = - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_CONNECTION, "setClientInfo not supported") - - override def setClientInfo(properties: Properties): Unit = - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_CONNECTION, "properties not supported") - - override def getClientInfo(name: String): String = - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_CONNECTION, "getClientInfo not supported") - - override def getClientInfo: Properties = - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_CONNECTION, "getClientInfo not supported") - - override def createArrayOf(typeName: String, elements: Array[AnyRef]): sql.Array = - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_CONNECTION, "createArrayOf not supported") - - override def createStruct(typeName: String, attributes: Array[AnyRef]): Struct = - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_CONNECTION, "createStruct not supported") - - override def setSchema(schema: String): Unit = throwWhenClosed { - if (StringUtils.isBlank(schema)) - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_STATEMENT, "schema is empty!") - createStatement().execute("use " + schema) - } - - override def getSchema: String = throwWhenClosed { - val resultSet = createStatement().executeQuery("SELECT current_database()") - if (!resultSet.next()) - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_STATEMENT, "Get schema failed!") - resultSet.getString(1) - } - - override def abort(executor: Executor): Unit = - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_CONNECTION, "abort not supported") - - override def setNetworkTimeout(executor: Executor, milliseconds: Int): Unit = - throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_CONNECTION, - "setNetworkTimeout not supported" - ) - - override def getNetworkTimeout: Int = throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_CONNECTION, - "getNetworkTimeout not supported" - ) - - override def unwrap[T](iface: Class[T]): T = - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_CONNECTION, "unwrap not supported") - - override def isWrapperFor(iface: Class[_]): Boolean = - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_CONNECTION, "isWrapperFor not supported") - -} diff --git a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLDatabaseMetaData.scala b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLDatabaseMetaData.scala index f2f0b9a106..25e94c5370 100644 --- a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLDatabaseMetaData.scala +++ b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLDatabaseMetaData.scala @@ -29,7 +29,7 @@ import java.util import scala.collection.JavaConversions._ -class UJESSQLDatabaseMetaData(ujesSQLConnection: UJESSQLConnection) +class UJESSQLDatabaseMetaData(ujesSQLConnection: LinkisSQLConnection) extends DatabaseMetaData with Logging { override def allProceduresAreCallable(): Boolean = false @@ -41,7 +41,8 @@ class UJESSQLDatabaseMetaData(ujesSQLConnection: UJESSQLConnection) override def getUserName: String = if (ujesSQLConnection.getProps.containsKey("user")) ujesSQLConnection.getProps.getProperty("user") - else throw new UJESSQLException(UJESSQLErrorCode.PARAMS_NOT_FOUND, "Missing user information") + else + throw new LinkisSQLException(LinkisSQLErrorCode.PARAMS_NOT_FOUND, "Missing user information") override def isReadOnly: Boolean = false @@ -88,7 +89,10 @@ class UJESSQLDatabaseMetaData(ujesSQLConnection: UJESSQLConnection) override def getIdentifierQuoteString: String = " " override def getSQLKeywords: String = - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_METADATA, "getSQLKeywords not supported") + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_METADATA, + "getSQLKeywords not supported" + ) override def getNumericFunctions: String = "" @@ -162,8 +166,8 @@ class UJESSQLDatabaseMetaData(ujesSQLConnection: UJESSQLConnection) override def getCatalogTerm: String = "instance" - override def isCatalogAtStart: Boolean = throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_METADATA, + override def isCatalogAtStart: Boolean = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_METADATA, "isCatalogAtStart not supported" ) @@ -219,103 +223,106 @@ class UJESSQLDatabaseMetaData(ujesSQLConnection: UJESSQLConnection) override def supportsOpenStatementsAcrossRollback(): Boolean = false - override def getMaxBinaryLiteralLength: Int = throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_METADATA, + override def getMaxBinaryLiteralLength: Int = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_METADATA, "getMaxBinaryLiteralLength not supported" ) - override def getMaxCharLiteralLength: Int = throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_METADATA, + override def getMaxCharLiteralLength: Int = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_METADATA, "getMaxCharLiteralLength not supported" ) override def getMaxColumnNameLength: Int = 128 - override def getMaxColumnsInGroupBy: Int = throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_METADATA, + override def getMaxColumnsInGroupBy: Int = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_METADATA, "getMaxColumnsInGroupBy not supported" ) - override def getMaxColumnsInIndex: Int = throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_METADATA, + override def getMaxColumnsInIndex: Int = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_METADATA, "getMaxColumnsInIndex not supported" ) - override def getMaxColumnsInOrderBy: Int = throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_METADATA, + override def getMaxColumnsInOrderBy: Int = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_METADATA, "getMaxColumnsInOrderBy not supported" ) - override def getMaxColumnsInSelect: Int = throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_METADATA, + override def getMaxColumnsInSelect: Int = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_METADATA, "getMaxColumnsInSelect not supported" ) - override def getMaxColumnsInTable: Int = throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_METADATA, + override def getMaxColumnsInTable: Int = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_METADATA, "getMaxColumnsInTable not supported" ) - override def getMaxConnections: Int = throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_METADATA, + override def getMaxConnections: Int = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_METADATA, "getMaxConnections not supported" ) - override def getMaxCursorNameLength: Int = throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_METADATA, + override def getMaxCursorNameLength: Int = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_METADATA, "getMaxCursorNameLength not supported" ) - override def getMaxIndexLength: Int = throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_METADATA, + override def getMaxIndexLength: Int = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_METADATA, "getMaxIndexLength not supported" ) - override def getMaxSchemaNameLength: Int = throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_METADATA, + override def getMaxSchemaNameLength: Int = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_METADATA, "getMaxSchemaNameLength not supported" ) - override def getMaxProcedureNameLength: Int = throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_METADATA, + override def getMaxProcedureNameLength: Int = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_METADATA, "getMaxProcedureNameLength not supported" ) - override def getMaxCatalogNameLength: Int = throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_METADATA, + override def getMaxCatalogNameLength: Int = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_METADATA, "getMaxCatalogNameLength not supported" ) override def getMaxRowSize: Int = - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_METADATA, "getMaxRowSize not supported") + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_METADATA, + "getMaxRowSize not supported" + ) - override def doesMaxRowSizeIncludeBlobs(): Boolean = throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_METADATA, + override def doesMaxRowSizeIncludeBlobs(): Boolean = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_METADATA, "doesMaxRowSizeIncludeBlobs not supported" ) - override def getMaxStatementLength: Int = throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_METADATA, + override def getMaxStatementLength: Int = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_METADATA, "getMaxStatementLength not supported" ) - override def getMaxStatements: Int = throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_METADATA, + override def getMaxStatements: Int = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_METADATA, "getMaxStatements not supported" ) - override def getMaxTableNameLength: Int = throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_METADATA, + override def getMaxTableNameLength: Int = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_METADATA, "getMaxTableNameLength not supported" ) - override def getMaxTablesInSelect: Int = throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_METADATA, + override def getMaxTablesInSelect: Int = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_METADATA, "getMaxTablesInSelect not supported" ) - override def getMaxUserNameLength: Int = throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_METADATA, + override def getMaxUserNameLength: Int = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_METADATA, "getMaxUserNameLength not supported" ) @@ -329,13 +336,13 @@ class UJESSQLDatabaseMetaData(ujesSQLConnection: UJESSQLConnection) override def supportsDataManipulationTransactionsOnly(): Boolean = false - override def dataDefinitionCausesTransactionCommit(): Boolean = throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_METADATA, + override def dataDefinitionCausesTransactionCommit(): Boolean = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_METADATA, "dataDefinitionCausesTransactionCommit not supported" ) - override def dataDefinitionIgnoredInTransactions(): Boolean = throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_METADATA, + override def dataDefinitionIgnoredInTransactions(): Boolean = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_METADATA, "dataDefinitionIgnoredInTransactions not supported" ) @@ -358,7 +365,9 @@ class UJESSQLDatabaseMetaData(ujesSQLConnection: UJESSQLConnection) tableNamePattern: String, types: Array[String] ): ResultSet = { - val resultCatalog = if (StringUtils.isNotBlank(catalog)) { + val resultCatalog = if (StringUtils.isNotBlank(schemaPattern)) { + schemaPattern + } else if (StringUtils.isNotBlank(catalog)) { catalog } else { s"${getUserName}_ind" @@ -374,11 +383,16 @@ class UJESSQLDatabaseMetaData(ujesSQLConnection: UJESSQLConnection) if (table.get("isView").asInstanceOf[Boolean]) TableType.VIEW.name() else TableType.TABLE.name() val resultTable = new util.HashMap[String, String]() + val tableName = table.get("tableName").asInstanceOf[String] resultTable.put("catalog", resultCatalog) - resultTable.put("tableName", table.get("tableName").asInstanceOf[String]) + resultTable.put("tableName", tableName) resultTable.put("tableType", tableType) if (null == types || types.contains(tableType)) { - resultTables.add(resultTable) + if ( + StringUtils.isNotBlank(tableNamePattern) && tableNamePattern.equalsIgnoreCase(tableName) + ) { + resultTables.add(resultTable) + } } } val resultSet: LinkisMetaDataResultSet[util.Map[String, String]] = @@ -471,7 +485,9 @@ class UJESSQLDatabaseMetaData(ujesSQLConnection: UJESSQLConnection) tableNamePattern: String, columnNamePattern: String ): ResultSet = { - val resultCatalog = if (StringUtils.isNotBlank(catalog)) { + val resultCatalog = if (StringUtils.isNotBlank(schemaPattern)) { + schemaPattern + } else if (StringUtils.isNotBlank(catalog)) { catalog } else { s"${getUserName}_ind" @@ -670,48 +686,48 @@ class UJESSQLDatabaseMetaData(ujesSQLConnection: UJESSQLConnection) override def supportsResultSetConcurrency(`type`: Int, concurrency: Int): Boolean = false - override def ownUpdatesAreVisible(`type`: Int): Boolean = throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_METADATA, + override def ownUpdatesAreVisible(`type`: Int): Boolean = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_METADATA, "ownUpdatesAreVisible not supported" ) - override def ownDeletesAreVisible(`type`: Int): Boolean = throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_METADATA, + override def ownDeletesAreVisible(`type`: Int): Boolean = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_METADATA, "ownDeletesAreVisible not supported" ) - override def ownInsertsAreVisible(`type`: Int): Boolean = throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_METADATA, + override def ownInsertsAreVisible(`type`: Int): Boolean = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_METADATA, "ownInsertsAreVisible not supported" ) - override def othersUpdatesAreVisible(`type`: Int): Boolean = throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_METADATA, + override def othersUpdatesAreVisible(`type`: Int): Boolean = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_METADATA, "othersUpdatesAreVisible not supported" ) - override def othersDeletesAreVisible(`type`: Int): Boolean = throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_METADATA, + override def othersDeletesAreVisible(`type`: Int): Boolean = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_METADATA, "othersDeletesAreVisible not supported" ) - override def othersInsertsAreVisible(`type`: Int): Boolean = throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_METADATA, + override def othersInsertsAreVisible(`type`: Int): Boolean = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_METADATA, "othersInsertsAreVisible not supported" ) - override def updatesAreDetected(`type`: Int): Boolean = throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_METADATA, + override def updatesAreDetected(`type`: Int): Boolean = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_METADATA, "updatesAreDetected not supported" ) - override def deletesAreDetected(`type`: Int): Boolean = throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_METADATA, + override def deletesAreDetected(`type`: Int): Boolean = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_METADATA, "deletesAreDetected not supported" ) - override def insertsAreDetected(`type`: Int): Boolean = throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_METADATA, + override def insertsAreDetected(`type`: Int): Boolean = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_METADATA, "insertsAreDetected not supported" ) @@ -739,14 +755,20 @@ class UJESSQLDatabaseMetaData(ujesSQLConnection: UJESSQLConnection) schemaPattern: String, typeNamePattern: String ): ResultSet = - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_METADATA, "getSuperTypes not supported") + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_METADATA, + "getSuperTypes not supported" + ) override def getSuperTables( catalog: String, schemaPattern: String, tableNamePattern: String ): ResultSet = - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_METADATA, "getSuperTables not supported") + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_METADATA, + "getSuperTables not supported" + ) override def getAttributes( catalog: String, @@ -754,12 +776,15 @@ class UJESSQLDatabaseMetaData(ujesSQLConnection: UJESSQLConnection) typeNamePattern: String, attributeNamePattern: String ): ResultSet = - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_METADATA, "getAttributes not supported") + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_METADATA, + "getAttributes not supported" + ) override def supportsResultSetHoldability(holdability: Int): Boolean = false - override def getResultSetHoldability: Int = throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_METADATA, + override def getResultSetHoldability: Int = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_METADATA, "getResultSetHoldability not supported" ) @@ -773,15 +798,15 @@ class UJESSQLDatabaseMetaData(ujesSQLConnection: UJESSQLConnection) override def getSQLStateType: Int = 2 - override def locatorsUpdateCopy(): Boolean = throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_METADATA, + override def locatorsUpdateCopy(): Boolean = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_METADATA, "locatorsUpdateCopy not supported" ) override def supportsStatementPooling(): Boolean = false - override def getRowIdLifetime: RowIdLifetime = throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_METADATA, + override def getRowIdLifetime: RowIdLifetime = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_METADATA, "getRowIdLifetime not supported" ) @@ -794,13 +819,13 @@ class UJESSQLDatabaseMetaData(ujesSQLConnection: UJESSQLConnection) override def supportsStoredFunctionsUsingCallSyntax(): Boolean = false - override def autoCommitFailureClosesAllResultSets(): Boolean = throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_METADATA, + override def autoCommitFailureClosesAllResultSets(): Boolean = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_METADATA, "autoCommitFailureClosesAllResultSets not supported" ) - override def getClientInfoProperties: ResultSet = throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_METADATA, + override def getClientInfoProperties: ResultSet = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_METADATA, "getClientInfoProperties not supported" ) @@ -824,15 +849,18 @@ class UJESSQLDatabaseMetaData(ujesSQLConnection: UJESSQLConnection) columnNamePattern: String ): ResultSet = null - override def generatedKeyAlwaysReturned(): Boolean = throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_METADATA, + override def generatedKeyAlwaysReturned(): Boolean = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_METADATA, "generatedKeyAlwaysReturned not supported" ) override def unwrap[T](iface: Class[T]): T = - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_METADATA, "unwrap not supported") + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_METADATA, "unwrap not supported") override def isWrapperFor(iface: Class[_]): Boolean = - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_METADATA, "isWrapperFor not supported") + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_METADATA, + "isWrapperFor not supported" + ) } diff --git a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLDriverMain.scala b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLDriverMain.scala index 783713cf40..ab2f6dda10 100644 --- a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLDriverMain.scala +++ b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLDriverMain.scala @@ -29,10 +29,10 @@ import java.sql.{ DriverPropertyInfo, SQLFeatureNotSupportedException } -import java.util.Properties +import java.util.{Locale, Properties} import java.util.logging.Logger -import scala.collection.JavaConversions +import scala.collection.JavaConverters._ class UJESSQLDriverMain extends Driver with Logging { @@ -41,8 +41,10 @@ class UJESSQLDriverMain extends Driver with Logging { props.putAll(parseURL(url)) logger.info(s"input url:$url, properties:$properties") val ujesClient = UJESClientFactory.getUJESClient(props) - new UJESSQLConnection(ujesClient, props) - } else throw new UJESSQLException(UJESSQLErrorCode.BAD_URL, "bad url: " + url) + new LinkisSQLConnection(ujesClient, props) + } else { + null + } override def acceptsURL(url: String): Boolean = url.startsWith(URL_PREFIX) @@ -70,20 +72,22 @@ class UJESSQLDriverMain extends Driver with Logging { case Array(TOKEN_VALUE, value) => props.setProperty(TOKEN_VALUE, value) false - case Array(LIMIT, value) => - props.setProperty(LIMIT, value) - UJESSQLDriverMain.LIMIT_ENABLED = value.toLowerCase() + case Array(FIXED_SESSION, value) => + props.setProperty(FIXED_SESSION, value) false case Array(key, _) => if (StringUtils.isBlank(key)) { - throw new UJESSQLException(UJESSQLErrorCode.BAD_URL, "bad url for params: " + url) + throw new LinkisSQLException( + LinkisSQLErrorCode.BAD_URL, + "bad url for params: " + url + ) } else true case _ => - throw new UJESSQLException(UJESSQLErrorCode.BAD_URL, "bad url for params: " + url) + throw new LinkisSQLException(LinkisSQLErrorCode.BAD_URL, "bad url for params: " + url) } props.setProperty(PARAMS, kvs.map(_.mkString(KV_SPLIT)).mkString(PARAM_SPLIT)) } - case _ => throw new UJESSQLException(UJESSQLErrorCode.BAD_URL, "bad url: " + url) + case _ => throw new LinkisSQLException(LinkisSQLErrorCode.BAD_URL, "bad url: " + url) } props } @@ -133,8 +137,7 @@ object UJESSQLDriverMain { val TOKEN_VALUE = UJESSQLDriver.TOKEN_VALUE val PASSWORD = UJESSQLDriver.PASSWORD val TABLEAU_SERVER = UJESSQLDriver.TABLEAU_SERVER - val LIMIT = UJESSQLDriver.LIMIT - var LIMIT_ENABLED = UJESSQLDriver.LIMIT_ENABLED + val FIXED_SESSION = UJESSQLDriver.FIXED_SESSION val VERSION = UJESSQLDriver.VERSION val DEFAULT_VERSION = UJESSQLDriver.DEFAULT_VERSION @@ -144,14 +147,15 @@ object UJESSQLDriverMain { val ENABLE_LOADBALANCER = UJESSQLDriver.ENABLE_LOADBALANCER val CREATOR = UJESSQLDriver.CREATOR + val TABLEAU = UJESSQLDriver.TABLEAU + val VARIABLE_HEADER = UJESSQLDriver.VARIABLE_HEADER def getConnectionParams( connectionParams: String, variableMap: java.util.Map[String, Any] ): String = { - val variables = JavaConversions - .mapAsScalaMap(variableMap) + val variables = variableMap.asScala .map(kv => VARIABLE_HEADER + kv._1 + KV_SPLIT + kv._2) .mkString(PARAM_SPLIT) if (StringUtils.isNotBlank(connectionParams)) connectionParams + PARAM_SPLIT + variables @@ -179,17 +183,20 @@ object UJESSQLDriverMain { ): String = { val sb = new StringBuilder if (StringUtils.isNotBlank(version)) sb.append(VERSION).append(KV_SPLIT).append(version) - if (maxConnectionSize > 0) + if (maxConnectionSize > 0) { sb.append(PARAM_SPLIT).append(MAX_CONNECTION_SIZE).append(KV_SPLIT).append(maxConnectionSize) - if (readTimeout > 0) + } + if (readTimeout > 0) { sb.append(PARAM_SPLIT).append(READ_TIMEOUT).append(KV_SPLIT).append(readTimeout) + } if (enableDiscovery) { sb.append(PARAM_SPLIT).append(ENABLE_DISCOVERY).append(KV_SPLIT).append(enableDiscovery) - if (enableLoadBalancer) + if (enableLoadBalancer) { sb.append(PARAM_SPLIT) .append(ENABLE_LOADBALANCER) .append(KV_SPLIT) .append(enableLoadBalancer) + } } if (sb.startsWith(PARAM_SPLIT)) sb.toString.substring(PARAM_SPLIT.length) else sb.toString } diff --git a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLException.scala b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLException.scala deleted file mode 100644 index 25db6f9381..0000000000 --- a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLException.scala +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.ujes.jdbc - -import org.apache.linkis.common.exception.ErrorException - -class UJESSQLException(errorCode: UJESSQLErrorCode) - extends ErrorException(errorCode.getCode, errorCode.getMsg) { - - def this(errorCode: UJESSQLErrorCode, msg: String) { - this(errorCode) - setErrCode(errorCode.getCode) - setDesc(msg) - } - - /** - * add to deal with errorinfo derived from jobInfo - * @param errorCode - * @param msg - */ - def this(errorCode: Int, msg: String) { - this(UJESSQLErrorCode.ERRORINFO_FROM_JOBINFO) - setDesc(msg) - setErrCode(errorCode) - } - -} diff --git a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLResultSet.scala b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLResultSet.scala index b8cf1b23b1..0ed47925c6 100644 --- a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLResultSet.scala +++ b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLResultSet.scala @@ -42,7 +42,7 @@ import java.sql.{ Time, Timestamp } -import java.util.Calendar +import java.util.{Calendar, Locale} import org.joda.time.DateTimeZone import org.joda.time.format.{ @@ -54,7 +54,7 @@ import org.joda.time.format.{ class UJESSQLResultSet( resultSetList: Array[String], - ujesStatement: UJESSQLStatement, + ujesStatement: LinkisSQLStatement, maxRows: Int, fetchSize: Int ) extends ResultSet @@ -75,10 +75,10 @@ class UJESSQLResultSet( private val pageSize: Int = 5000 private var path: String = _ private var metaData: util.List[util.Map[String, String]] = _ - private val statement: UJESSQLStatement = ujesStatement + private val statement: LinkisSQLStatement = ujesStatement - private val connection: UJESSQLConnection = - ujesStatement.getConnection.asInstanceOf[UJESSQLConnection] + private val connection: LinkisSQLConnection = + ujesStatement.getConnection.asInstanceOf[LinkisSQLConnection] private var valueWasNull: Boolean = false private var warningChain: SQLWarning = _ @@ -96,6 +96,10 @@ class UJESSQLResultSet( .toFormatter .withOffsetParsed + private val STRING_TYPE = "string" + + private val NULL_VALUE = "NULL" + private def getResultSetPath(resultSetList: Array[String]): String = { if (resultSetList.length > 0) { resultSetList(resultSetList.length - 1) @@ -189,7 +193,7 @@ class UJESSQLResultSet( if (metaData == null) init() currentRowCursor += 1 if (null == resultSetRow || currentRowCursor > resultSetRow.size() - 1) { - if (UJESSQLDriverMain.LIMIT_ENABLED.equals("false") && !isCompleted) { + if (!isCompleted) { updateResultSet() if (isCompleted) { return false @@ -229,12 +233,16 @@ class UJESSQLResultSet( } private def evaluate(dataType: String, value: String): Any = { + if (value == null || value.equals("null") || value.equals("NULL") || value.equals("Null")) { - value + dataType.toLowerCase(Locale.getDefault) match { + case "string" | "char" | "varchar" | "nvarchar" => value + case _ => null + } } else { - dataType.toLowerCase match { - case null => throw new UJESSQLException(UJESSQLErrorCode.METADATA_EMPTY) - case "string" => value.toString + dataType.toLowerCase(Locale.getDefault) match { + case null => throw new LinkisSQLException(LinkisSQLErrorCode.METADATA_EMPTY) + case "char" | "varchar" | "nvarchar" | "string" => value case "short" => value.toShort case "int" => value.toInt case "long" => value.toLong @@ -242,18 +250,15 @@ class UJESSQLResultSet( case "double" => value.toDouble case "boolean" => value.toBoolean case "byte" => value.toByte - case "char" => value.toString - case "timestamp" => value.toString - case "varchar" => value.toString - case "nvarchar" => value.toString - case "date" => value.toString + case "timestamp" => value + case "date" => value case "bigint" => value.toLong case "decimal" => value.toDouble case "array" => value.toArray case "map" => value case _ => - throw new UJESSQLException( - UJESSQLErrorCode.PREPARESTATEMENT_TYPEERROR, + throw new LinkisSQLException( + LinkisSQLErrorCode.PREPARESTATEMENT_TYPEERROR, s"Can't infer the SQL type to use for an instance of ${dataType}. Use getObject() with an explicit Types value to specify the type to use" ) } @@ -262,15 +267,15 @@ class UJESSQLResultSet( private def getColumnValue(columnIndex: Int): Any = { if (currentRow == null) { - throw new UJESSQLException(UJESSQLErrorCode.RESULTSET_ROWERROR, "No row found.") + throw new LinkisSQLException(LinkisSQLErrorCode.RESULTSET_ROWERROR, "No row found.") } else if (currentRow.size() <= 0) { - throw new UJESSQLException( - UJESSQLErrorCode.RESULTSET_ROWERROR, + throw new LinkisSQLException( + LinkisSQLErrorCode.RESULTSET_ROWERROR, "RowSet does not contain any columns!" ) } else if (columnIndex > currentRow.size()) { - throw new UJESSQLException( - UJESSQLErrorCode.RESULTSET_ROWERROR, + throw new LinkisSQLException( + LinkisSQLErrorCode.RESULTSET_ROWERROR, s" Invalid columnIndex: ${columnIndex}" ) } else { @@ -285,7 +290,7 @@ class UJESSQLResultSet( override def getString(columnIndex: Int): String = { val any = getColumnValue(columnIndex) if (wasNull()) { - throw new UJESSQLException(UJESSQLErrorCode.RESULTSET_ROWERROR, "Type is null") + throw new LinkisSQLException(LinkisSQLErrorCode.RESULTSET_ROWERROR, "Type is null") } else { any match { case c: Character => Character.toString(c) @@ -297,7 +302,7 @@ class UJESSQLResultSet( override def getBoolean(columnIndex: Int): Boolean = { val any = getColumnValue(columnIndex) if (wasNull()) { - throw new UJESSQLException(UJESSQLErrorCode.RESULTSET_ROWERROR, "Type is null") + throw new LinkisSQLException(LinkisSQLErrorCode.RESULTSET_ROWERROR, "Type is null") } else { any match { case s: String => @@ -313,7 +318,7 @@ class UJESSQLResultSet( override def getByte(columnIndex: Int): Byte = { val any = getColumnValue(columnIndex) if (wasNull()) { - throw new UJESSQLException(UJESSQLErrorCode.RESULTSET_ROWERROR, "Type is null") + throw new LinkisSQLException(LinkisSQLErrorCode.RESULTSET_ROWERROR, "Type is null") } else { any.asInstanceOf[Byte] } @@ -322,7 +327,7 @@ class UJESSQLResultSet( override def getShort(columnIndex: Int): Short = { val any = getColumnValue(columnIndex) if (wasNull()) { - throw new UJESSQLException(UJESSQLErrorCode.RESULTSET_ROWERROR, "Type is null") + throw new LinkisSQLException(LinkisSQLErrorCode.RESULTSET_ROWERROR, "Type is null") } else { any.asInstanceOf[Short] } @@ -331,7 +336,7 @@ class UJESSQLResultSet( override def getInt(columnIndex: Int): Int = { val any = getColumnValue(columnIndex) if (wasNull()) { - throw new UJESSQLException(UJESSQLErrorCode.RESULTSET_ROWERROR, "Type is null") + throw new LinkisSQLException(LinkisSQLErrorCode.RESULTSET_ROWERROR, "Type is null") } else { any match { case i: Integer => i.asInstanceOf[Int] @@ -344,7 +349,7 @@ class UJESSQLResultSet( override def getLong(columnIndex: Int): Long = { val any = getColumnValue(columnIndex) if (wasNull()) { - throw new UJESSQLException(UJESSQLErrorCode.RESULTSET_ROWERROR, "Type is null") + throw new LinkisSQLException(LinkisSQLErrorCode.RESULTSET_ROWERROR, "Type is null") } else { any match { case i: Integer => i.longValue() @@ -357,7 +362,7 @@ class UJESSQLResultSet( override def getFloat(columnIndex: Int): Float = { val any = getColumnValue(columnIndex) if (wasNull()) { - throw new UJESSQLException(UJESSQLErrorCode.RESULTSET_ROWERROR, "Type is null") + throw new LinkisSQLException(LinkisSQLErrorCode.RESULTSET_ROWERROR, "Type is null") } else { any.asInstanceOf[Float] } @@ -366,7 +371,7 @@ class UJESSQLResultSet( override def getDouble(columnIndex: Int): Double = { val any = getColumnValue(columnIndex) if (wasNull()) { - throw new UJESSQLException(UJESSQLErrorCode.RESULTSET_ROWERROR, "Type is null") + throw new LinkisSQLException(LinkisSQLErrorCode.RESULTSET_ROWERROR, "Type is null") } else { any match { case _: String => 0.0d @@ -379,7 +384,7 @@ class UJESSQLResultSet( val mc = new MathContext(scale) val any = getColumnValue(columnIndex) if (wasNull()) { - throw new UJESSQLException(UJESSQLErrorCode.RESULTSET_ROWERROR, "Type is null") + throw new LinkisSQLException(LinkisSQLErrorCode.RESULTSET_ROWERROR, "Type is null") } else { any match { case double: Double => new java.math.BigDecimal(double).round(mc) @@ -393,7 +398,7 @@ class UJESSQLResultSet( override def getBytes(columnIndex: Int): Array[Byte] = { val any = getColumnValue(columnIndex) if (wasNull()) { - throw new UJESSQLException(UJESSQLErrorCode.RESULTSET_ROWERROR, "Type is null") + throw new LinkisSQLException(LinkisSQLErrorCode.RESULTSET_ROWERROR, "Type is null") } else { any.asInstanceOf[Array[Byte]] } @@ -404,35 +409,35 @@ class UJESSQLResultSet( val any = getColumnValue(columnIndex) logger.info(s"the value of Date is $any") if (wasNull()) { - throw new UJESSQLException(UJESSQLErrorCode.RESULTSET_ROWERROR, "Type is null") + throw new LinkisSQLException(LinkisSQLErrorCode.RESULTSET_ROWERROR, "Type is null") } else { any.asInstanceOf[Date] } } override def getTime(columnIndex: Int): Time = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def getTimestamp(columnIndex: Int): Timestamp = { val any = getColumnValue(columnIndex) if (wasNull()) { - throw new UJESSQLException(UJESSQLErrorCode.RESULTSET_ROWERROR, "Type is null") + throw new LinkisSQLException(LinkisSQLErrorCode.RESULTSET_ROWERROR, "Type is null") } else { any.asInstanceOf[Timestamp] } } override def getAsciiStream(columnIndex: Int): InputStream = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def getUnicodeStream(columnIndex: Int): InputStream = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def getBinaryStream(columnIndex: Int): InputStream = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def getString(columnLabel: String): String = { @@ -480,7 +485,7 @@ class UJESSQLResultSet( } override def getTime(columnLabel: String): Time = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def getTimestamp(columnLabel: String): Timestamp = { @@ -488,15 +493,15 @@ class UJESSQLResultSet( } override def getAsciiStream(columnLabel: String): InputStream = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def getUnicodeStream(columnLabel: String): InputStream = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def getBinaryStream(columnLabel: String): InputStream = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def getWarnings: SQLWarning = { @@ -508,7 +513,7 @@ class UJESSQLResultSet( } override def getCursorName: String = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def getMetaData: UJESSQLResultSetMetaData = { @@ -518,11 +523,7 @@ class UJESSQLResultSet( override def getObject(columnIndex: Int): Object = { val any = getColumnValue(columnIndex) - if (wasNull()) { - throw new UJESSQLException(UJESSQLErrorCode.RESULTSET_ROWERROR, "Type is null") - } else { - any.asInstanceOf[Object] - } + any.asInstanceOf[Object] } override def getObject(columnLabel: String): Object = { @@ -542,19 +543,19 @@ class UJESSQLResultSet( } } if (columnIndex == -1) { - throw new UJESSQLException( - UJESSQLErrorCode.RESULTSET_ROWERROR, + throw new LinkisSQLException( + LinkisSQLErrorCode.RESULTSET_ROWERROR, s"can not find column: ${columnLabel}" ) } else columnIndex } override def getCharacterStream(columnIndex: Int): Reader = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def getCharacterStream(columnLabel: String): Reader = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def getBigDecimal(columnIndex: Int): java.math.BigDecimal = { @@ -567,31 +568,31 @@ class UJESSQLResultSet( override def isBeforeFirst: Boolean = { if (resultSetRow == null) { - throw new UJESSQLException(UJESSQLErrorCode.RESULTSET_NULL) + throw new LinkisSQLException(LinkisSQLErrorCode.RESULTSET_NULL) } else currentRowCursor == -1 } override def isAfterLast: Boolean = { if (resultSetRow == null) { - throw new UJESSQLException(UJESSQLErrorCode.RESULTSET_NULL) + throw new LinkisSQLException(LinkisSQLErrorCode.RESULTSET_NULL) } else currentRowCursor > resultSetRow.size() - 1 } override def isFirst: Boolean = { if (resultSetRow == null) { - throw new UJESSQLException(UJESSQLErrorCode.RESULTSET_NULL) + throw new LinkisSQLException(LinkisSQLErrorCode.RESULTSET_NULL) } else currentRowCursor == 0 } override def isLast: Boolean = { if (resultSetRow == null) { - throw new UJESSQLException(UJESSQLErrorCode.RESULTSET_NULL) + throw new LinkisSQLException(LinkisSQLErrorCode.RESULTSET_NULL) } else currentRowCursor == resultSetRow.size() - 1 } override def beforeFirst(): Unit = { if (resultSetRow == null) { - throw new UJESSQLException(UJESSQLErrorCode.RESULTSET_NULL) + throw new LinkisSQLException(LinkisSQLErrorCode.RESULTSET_NULL) } else { currentRowCursor = -1 updateCurrentRow(currentRowCursor) @@ -600,7 +601,7 @@ class UJESSQLResultSet( override def afterLast(): Unit = { if (resultSetRow == null) { - throw new UJESSQLException(UJESSQLErrorCode.RESULTSET_NULL) + throw new LinkisSQLException(LinkisSQLErrorCode.RESULTSET_NULL) } else { currentRowCursor = resultSetRow.size() updateCurrentRow(currentRowCursor) @@ -627,7 +628,7 @@ class UJESSQLResultSet( override def getRow: Int = { if (resultSetRow == null) { - throw new UJESSQLException(UJESSQLErrorCode.RESULTSET_NULL) + throw new LinkisSQLException(LinkisSQLErrorCode.RESULTSET_NULL) } else { currentRowCursor + 1 } @@ -635,10 +636,10 @@ class UJESSQLResultSet( override def absolute(row: Int): Boolean = { if (resultSetRow == null) { - throw new UJESSQLException(UJESSQLErrorCode.RESULTSET_NULL) + throw new LinkisSQLException(LinkisSQLErrorCode.RESULTSET_NULL) } else if (row > resultSetRow.size()) { - throw new UJESSQLException( - UJESSQLErrorCode.RESULTSET_ROWERROR, + throw new LinkisSQLException( + LinkisSQLErrorCode.RESULTSET_ROWERROR, "The specified number of rows is greater than the maximum number of rows" ) } else { @@ -654,10 +655,10 @@ class UJESSQLResultSet( override def relative(rows: Int): Boolean = { if (resultSetRow == null) { - throw new UJESSQLException(UJESSQLErrorCode.RESULTSET_NULL) + throw new LinkisSQLException(LinkisSQLErrorCode.RESULTSET_NULL) } else if (rows > resultSetRow.size()) { - throw new UJESSQLException( - UJESSQLErrorCode.RESULTSET_ROWERROR, + throw new LinkisSQLException( + LinkisSQLErrorCode.RESULTSET_ROWERROR, "The specified number of rows is greater than the maximum number of rows" ) } else { @@ -679,11 +680,11 @@ class UJESSQLResultSet( } override def setFetchDirection(direction: Int): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def getFetchDirection: Int = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def setFetchSize(rows: Int): Unit = { @@ -703,248 +704,248 @@ class UJESSQLResultSet( } override def rowUpdated(): Boolean = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def rowInserted(): Boolean = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def rowDeleted(): Boolean = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateNull(columnIndex: Int): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateBoolean(columnIndex: Int, x: Boolean): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateByte(columnIndex: Int, x: Byte): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateShort(columnIndex: Int, x: Short): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateInt(columnIndex: Int, x: Int): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateLong(columnIndex: Int, x: Long): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateFloat(columnIndex: Int, x: Float): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateDouble(columnIndex: Int, x: Double): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateBigDecimal(columnIndex: Int, x: java.math.BigDecimal): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateString(columnIndex: Int, x: String): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateBytes(columnIndex: Int, x: Array[Byte]): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateDate(columnIndex: Int, x: Date): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateTime(columnIndex: Int, x: Time): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateTimestamp(columnIndex: Int, x: Timestamp): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateAsciiStream(columnIndex: Int, x: InputStream, length: Int): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateBinaryStream(columnIndex: Int, x: InputStream, length: Int): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateCharacterStream(columnIndex: Int, x: Reader, length: Int): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateObject(columnIndex: Int, x: scala.Any, scaleOrLength: Int): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateObject(columnIndex: Int, x: scala.Any): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateNull(columnLabel: String): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateBoolean(columnLabel: String, x: Boolean): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateByte(columnLabel: String, x: Byte): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateShort(columnLabel: String, x: Short): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateInt(columnLabel: String, x: Int): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateLong(columnLabel: String, x: Long): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateFloat(columnLabel: String, x: Float): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateDouble(columnLabel: String, x: Double): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateBigDecimal(columnLabel: String, x: java.math.BigDecimal): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateString(columnLabel: String, x: String): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateBytes(columnLabel: String, x: Array[Byte]): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateDate(columnLabel: String, x: Date): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateTime(columnLabel: String, x: Time): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateTimestamp(columnLabel: String, x: Timestamp): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateAsciiStream(columnLabel: String, x: InputStream, length: Int): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateBinaryStream(columnLabel: String, x: InputStream, length: Int): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateCharacterStream(columnLabel: String, reader: Reader, length: Int): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateObject(columnLabel: String, x: scala.Any, scaleOrLength: Int): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateObject(columnLabel: String, x: scala.Any): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def insertRow(): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateRow(): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def deleteRow(): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def refreshRow(): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def cancelRowUpdates(): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def moveToInsertRow(): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def moveToCurrentRow(): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def getStatement: Statement = { if (statement != null && !hasClosed) { statement.asInstanceOf[Statement] - } else throw new UJESSQLException(UJESSQLErrorCode.STATEMENT_CLOSED) + } else throw new LinkisSQLException(LinkisSQLErrorCode.STATEMENT_CLOSED) } override def getObject(columnIndex: Int, map: util.Map[String, Class[_]]): AnyRef = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def getRef(columnIndex: Int): Ref = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def getBlob(columnIndex: Int): Blob = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def getClob(columnIndex: Int): Clob = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def getArray(columnIndex: Int): sql.Array = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def getObject(columnLabel: String, map: util.Map[String, Class[_]]): AnyRef = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def getRef(columnLabel: String): Ref = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def getBlob(columnLabel: String): Blob = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def getClob(columnLabel: String): Clob = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def getArray(columnLabel: String): sql.Array = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } private def getDate(columnIndex: Int, localTimeZone: DateTimeZone): Date = { val value = getColumnValue(columnIndex) logger.info(s"the value of value is $value and the value of localTimeZone is $localTimeZone") if (wasNull()) { - throw new UJESSQLException(UJESSQLErrorCode.RESULTSET_ROWERROR, "Type is null") + throw new LinkisSQLException(LinkisSQLErrorCode.RESULTSET_ROWERROR, "Type is null") } else new Date(DATE_FORMATTER.withZone(localTimeZone).parseMillis(String.valueOf(value))); } @@ -958,11 +959,11 @@ class UJESSQLResultSet( } override def getTime(columnIndex: Int, cal: Calendar): Time = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def getTime(columnLabel: String, cal: Calendar): Time = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } private def getTimestamp(columnIndex: Int, localTimeZone: DateTimeZone): Timestamp = { @@ -970,8 +971,9 @@ class UJESSQLResultSet( logger.info(s"the value of value is $value and the value of localTimeZone is $localTimeZone") if (wasNull()) { null - } else + } else { new Timestamp(TIMESTAMP_FORMATTER.withZone(localTimeZone).parseMillis(String.valueOf(value))) + } } override def getTimestamp(columnIndex: Int, cal: Calendar): Timestamp = { @@ -986,63 +988,63 @@ class UJESSQLResultSet( } override def getURL(columnIndex: Int): URL = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def getURL(columnLabel: String): URL = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateRef(columnIndex: Int, x: Ref): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateRef(columnLabel: String, x: Ref): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateBlob(columnIndex: Int, x: Blob): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateBlob(columnLabel: String, x: Blob): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateClob(columnIndex: Int, x: Clob): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateClob(columnLabel: String, x: Clob): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateArray(columnIndex: Int, x: sql.Array): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateArray(columnLabel: String, x: sql.Array): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def getRowId(columnIndex: Int): RowId = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def getRowId(columnLabel: String): RowId = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateRowId(columnIndex: Int, x: RowId): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateRowId(columnLabel: String, x: RowId): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def getHoldability: Int = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def isClosed: Boolean = { @@ -1050,187 +1052,187 @@ class UJESSQLResultSet( } override def updateNString(columnIndex: Int, nString: String): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateNString(columnLabel: String, nString: String): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateNClob(columnIndex: Int, nClob: NClob): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateNClob(columnLabel: String, nClob: NClob): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def getNClob(columnIndex: Int): NClob = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def getNClob(columnLabel: String): NClob = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def getSQLXML(columnIndex: Int): SQLXML = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def getSQLXML(columnLabel: String): SQLXML = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateSQLXML(columnIndex: Int, xmlObject: SQLXML): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateSQLXML(columnLabel: String, xmlObject: SQLXML): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def getNString(columnIndex: Int): String = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def getNString(columnLabel: String): String = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def getNCharacterStream(columnIndex: Int): Reader = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def getNCharacterStream(columnLabel: String): Reader = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateNCharacterStream(columnIndex: Int, x: Reader, length: Long): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateNCharacterStream(columnLabel: String, reader: Reader, length: Long): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateAsciiStream(columnIndex: Int, x: InputStream, length: Long): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateBinaryStream(columnIndex: Int, x: InputStream, length: Long): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateCharacterStream(columnIndex: Int, x: Reader, length: Long): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateAsciiStream(columnLabel: String, x: InputStream, length: Long): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateBinaryStream(columnLabel: String, x: InputStream, length: Long): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateCharacterStream(columnLabel: String, reader: Reader, length: Long): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateBlob(columnIndex: Int, inputStream: InputStream, length: Long): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateBlob(columnLabel: String, inputStream: InputStream, length: Long): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateClob(columnIndex: Int, reader: Reader, length: Long): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateClob(columnLabel: String, reader: Reader, length: Long): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateNClob(columnIndex: Int, reader: Reader, length: Long): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateNClob(columnLabel: String, reader: Reader, length: Long): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateNCharacterStream(columnIndex: Int, x: Reader): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateNCharacterStream(columnLabel: String, reader: Reader): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateAsciiStream(columnIndex: Int, x: InputStream): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateBinaryStream(columnIndex: Int, x: InputStream): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateCharacterStream(columnIndex: Int, x: Reader): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateAsciiStream(columnLabel: String, x: InputStream): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateBinaryStream(columnLabel: String, x: InputStream): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateCharacterStream(columnLabel: String, reader: Reader): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateBlob(columnIndex: Int, inputStream: InputStream): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateBlob(columnLabel: String, inputStream: InputStream): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateClob(columnIndex: Int, reader: Reader): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateClob(columnLabel: String, reader: Reader): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateNClob(columnIndex: Int, reader: Reader): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateNClob(columnLabel: String, reader: Reader): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def getObject[T](columnIndex: Int, `type`: Class[T]): T = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def getObject[T](columnLabel: String, `type`: Class[T]): T = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def unwrap[T](iface: Class[T]): T = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def isWrapperFor(iface: Class[_]): Boolean = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } } diff --git a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLResultSetMetaData.scala b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLResultSetMetaData.scala index e51309e885..eaeb1b25d8 100644 --- a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLResultSetMetaData.scala +++ b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLResultSetMetaData.scala @@ -33,19 +33,19 @@ class UJESSQLResultSetMetaData extends ResultSetMetaData with Logging { private[jdbc] def setColumnNameProperties(column: Int, columnName: String): Unit = { if (column != null && columnName != null) { columnNameProperties.put(column, columnName) - } else throw new UJESSQLException(UJESSQLErrorCode.METADATA_EMPTY) + } else throw new LinkisSQLException(LinkisSQLErrorCode.METADATA_EMPTY) } private[jdbc] def setDataTypeProperties(column: Int, columnName: String): Unit = { if (column != null && columnName != null) { dataTypeProperties.put(column, columnName) - } else throw new UJESSQLException(UJESSQLErrorCode.METADATA_EMPTY) + } else throw new LinkisSQLException(LinkisSQLErrorCode.METADATA_EMPTY) } private[jdbc] def setCommentPropreties(column: Int, columnName: String): Unit = { if (column != null && columnName != null) { commentProperties.put(column, columnName) - } else throw new UJESSQLException(UJESSQLErrorCode.METADATA_EMPTY) + } else throw new LinkisSQLException(LinkisSQLErrorCode.METADATA_EMPTY) } override def getColumnCount: Int = { @@ -53,7 +53,7 @@ class UJESSQLResultSetMetaData extends ResultSetMetaData with Logging { } override def isAutoIncrement(column: Int): Boolean = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_METADATA) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_METADATA) } override def isCaseSensitive(column: Int): Boolean = true @@ -74,7 +74,7 @@ class UJESSQLResultSetMetaData extends ResultSetMetaData with Logging { override def getColumnLabel(column: Int): String = { if (columnNameProperties.get(column) == null) { - throw new UJESSQLException(UJESSQLErrorCode.METADATA_EMPTY) + throw new LinkisSQLException(LinkisSQLErrorCode.METADATA_EMPTY) } else columnNameProperties.get(column) } @@ -83,7 +83,7 @@ class UJESSQLResultSetMetaData extends ResultSetMetaData with Logging { } override def getSchemaName(column: Int): String = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_METADATA) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_METADATA) } override def getPrecision(column: Int): Int = { @@ -102,11 +102,11 @@ class UJESSQLResultSetMetaData extends ResultSetMetaData with Logging { } override def getTableName(column: Int): String = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_METADATA) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_METADATA) } override def getCatalogName(column: Int): String = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_METADATA) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_METADATA) } override def getColumnType(column: Int): Int = { @@ -115,7 +115,7 @@ class UJESSQLResultSetMetaData extends ResultSetMetaData with Logging { override def getColumnTypeName(column: Int): String = { if (dataTypeProperties.get(column) == null) { - throw new UJESSQLException(UJESSQLErrorCode.METADATA_EMPTY) + throw new LinkisSQLException(LinkisSQLErrorCode.METADATA_EMPTY) } else dataTypeProperties.get(column) } @@ -128,19 +128,19 @@ class UJESSQLResultSetMetaData extends ResultSetMetaData with Logging { } override def isDefinitelyWritable(column: Int): Boolean = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_METADATA) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_METADATA) } override def getColumnClassName(column: Int): String = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_METADATA) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_METADATA) } override def unwrap[T](iface: Class[T]): T = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_METADATA) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_METADATA) } override def isWrapperFor(iface: Class[_]): Boolean = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_METADATA) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_METADATA) } protected def toZeroIndex(column: Int): Int = { diff --git a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLTypeParser.scala b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLTypeParser.scala index ba4cd3878f..d8de812a1b 100644 --- a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLTypeParser.scala +++ b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLTypeParser.scala @@ -18,12 +18,14 @@ package org.apache.linkis.ujes.jdbc import java.sql.{SQLException, Timestamp, Types} +import java.util.Locale object UJESSQLTypeParser { def parserFromName(typeName: String): Int = { - typeName.toLowerCase match { - case null => throw new UJESSQLException(UJESSQLErrorCode.METADATA_EMPTY) + val typeNameLowerCase = typeName.toLowerCase(Locale.getDefault()) + typeName.toLowerCase() match { + case null => throw new LinkisSQLException(LinkisSQLErrorCode.METADATA_EMPTY) case "string" => Types.NVARCHAR case "short" => Types.SMALLINT case "int" => Types.INTEGER @@ -43,7 +45,12 @@ object UJESSQLTypeParser { case "bigint" => Types.BIGINT case "array" => Types.ARRAY case "map" => Types.JAVA_OBJECT - case _ => throw new SQLException(s"parameter type error,Type:$typeName") + case _ => + if (typeNameLowerCase.startsWith("decimal")) { + Types.DECIMAL + } else { + Types.NVARCHAR + } } } @@ -60,7 +67,7 @@ object UJESSQLTypeParser { case _: Char => Types.CHAR case _: BigDecimal => Types.DECIMAL case _: Timestamp => Types.TIMESTAMP - case _ => throw new UJESSQLException(UJESSQLErrorCode.PREPARESTATEMENT_TYPEERROR) + case _ => throw new LinkisSQLException(LinkisSQLErrorCode.PREPARESTATEMENT_TYPEERROR) } } @@ -80,7 +87,7 @@ object UJESSQLTypeParser { case Types.VARCHAR => "varchar" case Types.NVARCHAR => "string" case Types.DATE => "date" - case _ => throw new UJESSQLException(UJESSQLErrorCode.PREPARESTATEMENT_TYPEERROR) + case _ => throw new LinkisSQLException(LinkisSQLErrorCode.PREPARESTATEMENT_TYPEERROR) } } diff --git a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/hook/JDBCDriverPreExecutionHook.scala b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/hook/JDBCDriverPreExecutionHook.scala index 038ff38712..c7de7d3734 100644 --- a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/hook/JDBCDriverPreExecutionHook.scala +++ b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/hook/JDBCDriverPreExecutionHook.scala @@ -24,7 +24,7 @@ import scala.collection.mutable.ArrayBuffer trait JDBCDriverPreExecutionHook { - def callPreExecutionHook(sql: String): String + def callPreExecutionHook(sql: String, skip: Boolean): String } @@ -34,8 +34,7 @@ object JDBCDriverPreExecutionHook extends Logging { val hooks = new ArrayBuffer[JDBCDriverPreExecutionHook]() CommonVars( "wds.linkis.jdbc.pre.hook.class", - "org.apache.linkis.ujes.jdbc.hook.impl.TableauPreExecutionHook," + - "org.apache.linkis.ujes.jdbc.hook.impl.NoLimitExecutionHook" + "org.apache.linkis.ujes.jdbc.hook.impl.TableauPreExecutionHook" ).getValue.split(",") foreach { hookStr => Utils.tryCatch { val clazz = Class.forName(hookStr.trim) @@ -51,5 +50,5 @@ object JDBCDriverPreExecutionHook extends Logging { hooks.toArray } - def getPreExecutionHooks = preExecutionHooks + def getPreExecutionHooks: Array[JDBCDriverPreExecutionHook] = preExecutionHooks } diff --git a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/hook/impl/NoLimitExecutionHook.scala b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/hook/impl/NoLimitExecutionHook.scala deleted file mode 100644 index 18f94c0fde..0000000000 --- a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/hook/impl/NoLimitExecutionHook.scala +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.ujes.jdbc.hook.impl - -import org.apache.linkis.ujes.jdbc.UJESSQLDriverMain -import org.apache.linkis.ujes.jdbc.hook.JDBCDriverPreExecutionHook - -class NoLimitExecutionHook extends JDBCDriverPreExecutionHook { - - override def callPreExecutionHook(sql: String): String = { - if (UJESSQLDriverMain.LIMIT_ENABLED.toLowerCase.equals("false")) { - var noLimitSql = "--set ide.engine.no.limit.allow=true\n" + sql - val lowerCaseLimitSql = noLimitSql.toLowerCase() - if (lowerCaseLimitSql.contains("limit ") && lowerCaseLimitSql.contains("tableausql")) { - val lastIndexOfLimit = lowerCaseLimitSql.lastIndexOf("limit ") - noLimitSql = noLimitSql.substring(0, lastIndexOfLimit) - } - noLimitSql - } else { - sql - } - - } - -} diff --git a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/hook/impl/TableauPreExecutionHook.scala b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/hook/impl/TableauPreExecutionHook.scala index 63c4f53c9f..ec177b4de7 100644 --- a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/hook/impl/TableauPreExecutionHook.scala +++ b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/hook/impl/TableauPreExecutionHook.scala @@ -21,7 +21,10 @@ import org.apache.linkis.ujes.jdbc.hook.JDBCDriverPreExecutionHook class TableauPreExecutionHook extends JDBCDriverPreExecutionHook { - override def callPreExecutionHook(sql: String): String = { + override def callPreExecutionHook(sql: String, skip: Boolean): String = { + if (skip) { + return sql + } if ( sql.contains("CREATE INDEX") || sql .contains("CREATE TABLE") || sql.contains("INSERT INTO") || sql.contains("DROP TABLE") diff --git a/linkis-computation-governance/linkis-jdbc-driver/src/test/java/org/apache/linkis/ujes/jdbc/CreateConnection.java b/linkis-computation-governance/linkis-jdbc-driver/src/test/java/org/apache/linkis/ujes/jdbc/CreateConnection.java index 51d9e9953a..ca2c1e35fd 100644 --- a/linkis-computation-governance/linkis-jdbc-driver/src/test/java/org/apache/linkis/ujes/jdbc/CreateConnection.java +++ b/linkis-computation-governance/linkis-jdbc-driver/src/test/java/org/apache/linkis/ujes/jdbc/CreateConnection.java @@ -27,12 +27,12 @@ public class CreateConnection { - private static UJESSQLConnection conn; + private static LinkisSQLConnection conn; - public static UJESSQLConnection getConnection() throws ClassNotFoundException, SQLException { + public static LinkisSQLConnection getConnection() throws ClassNotFoundException, SQLException { Class.forName("org.apache.linkis.ujes.jdbc.UJESSQLDriver"); conn = - (UJESSQLConnection) + (LinkisSQLConnection) DriverManager.getConnection("jdbc:linkis://hostname:port", "username", "password"); return conn; } diff --git a/linkis-computation-governance/linkis-jdbc-driver/src/test/java/org/apache/linkis/ujes/jdbc/JDBCSpiTest.java b/linkis-computation-governance/linkis-jdbc-driver/src/test/java/org/apache/linkis/ujes/jdbc/JDBCSpiTest.java index 427a6b5fc4..3e76bb2856 100644 --- a/linkis-computation-governance/linkis-jdbc-driver/src/test/java/org/apache/linkis/ujes/jdbc/JDBCSpiTest.java +++ b/linkis-computation-governance/linkis-jdbc-driver/src/test/java/org/apache/linkis/ujes/jdbc/JDBCSpiTest.java @@ -29,21 +29,21 @@ * */ public class JDBCSpiTest { - private static UJESSQLConnection conn; + private static LinkisSQLConnection conn; - public static UJESSQLConnection getConnection() throws ClassNotFoundException, SQLException { + public static LinkisSQLConnection getConnection() throws ClassNotFoundException, SQLException { Class.forName("org.apache.linkis.ujes.jdbc.UJESSQLDriver"); conn = - (UJESSQLConnection) - DriverManager.getConnection("jdbc:linkis://hostname:port", "root", "123456"); + (LinkisSQLConnection) + DriverManager.getConnection("jdbc:linkis://127.0.0.1:9001", "root", "123456"); return conn; } @Test public void spiTest() { try { - UJESSQLConnection conn = - (UJESSQLConnection) + LinkisSQLConnection conn = + (LinkisSQLConnection) DriverManager.getConnection("jdbc:linkis://hostname:port", "username", "password"); Assertions.assertNotNull(conn); } catch (SQLException e) { diff --git a/linkis-computation-governance/linkis-jdbc-driver/src/test/java/org/apache/linkis/ujes/jdbc/UJESSQLStatementTest.java b/linkis-computation-governance/linkis-jdbc-driver/src/test/java/org/apache/linkis/ujes/jdbc/LinkisSQLStatementTest.java similarity index 95% rename from linkis-computation-governance/linkis-jdbc-driver/src/test/java/org/apache/linkis/ujes/jdbc/UJESSQLStatementTest.java rename to linkis-computation-governance/linkis-jdbc-driver/src/test/java/org/apache/linkis/ujes/jdbc/LinkisSQLStatementTest.java index e674dd5b23..3ebd21ae70 100644 --- a/linkis-computation-governance/linkis-jdbc-driver/src/test/java/org/apache/linkis/ujes/jdbc/UJESSQLStatementTest.java +++ b/linkis-computation-governance/linkis-jdbc-driver/src/test/java/org/apache/linkis/ujes/jdbc/LinkisSQLStatementTest.java @@ -33,9 +33,9 @@ * if you want to test this module,you must rewrite default parameters and SQL we used for local test * */ -public class UJESSQLStatementTest { - private static UJESSQLConnection conn; - private static UJESSQLStatement statement; +public class LinkisSQLStatementTest { + private static LinkisSQLConnection conn; + private static LinkisSQLStatement statement; private static int maxRows; private static int queryTimeout; private static String sql; @@ -48,7 +48,7 @@ public class UJESSQLStatementTest { public static void createConnection() { try { conn = CreateConnection.getConnection(); - statement = (UJESSQLStatement) conn.createStatement(); + statement = (LinkisSQLStatement) conn.createStatement(); } catch (ClassNotFoundException e) { e.printStackTrace(); } catch (SQLException e) { diff --git a/linkis-computation-governance/linkis-jdbc-driver/src/test/java/org/apache/linkis/ujes/jdbc/UJESSQLDatabaseMetaDataTest.java b/linkis-computation-governance/linkis-jdbc-driver/src/test/java/org/apache/linkis/ujes/jdbc/UJESSQLDatabaseMetaDataTest.java index 0dab63b3ff..100e13f3e4 100644 --- a/linkis-computation-governance/linkis-jdbc-driver/src/test/java/org/apache/linkis/ujes/jdbc/UJESSQLDatabaseMetaDataTest.java +++ b/linkis-computation-governance/linkis-jdbc-driver/src/test/java/org/apache/linkis/ujes/jdbc/UJESSQLDatabaseMetaDataTest.java @@ -34,7 +34,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; public class UJESSQLDatabaseMetaDataTest { - private static UJESSQLConnection conn; + private static LinkisSQLConnection conn; private static UJESSQLDatabaseMetaData dbmd; @BeforeAll @@ -54,42 +54,43 @@ public static void preWork() { @Test public void supportsMinimumSQLGrammar() { if (dbmd != null) { - Assertions.assertThrows(UJESSQLException.class, () -> dbmd.supportsMinimumSQLGrammar()); + Assertions.assertThrows(LinkisSQLException.class, () -> dbmd.supportsMinimumSQLGrammar()); } } @Test public void getResultSetHoldability() { if (dbmd != null) { - Assertions.assertThrows(UJESSQLException.class, () -> dbmd.getResultSetHoldability()); + Assertions.assertThrows(LinkisSQLException.class, () -> dbmd.getResultSetHoldability()); } } @Test public void getMaxColumnsInGroupBy() { if (dbmd != null) { - Assertions.assertThrows(UJESSQLException.class, () -> dbmd.getMaxColumnsInGroupBy()); + Assertions.assertThrows(LinkisSQLException.class, () -> dbmd.getMaxColumnsInGroupBy()); } } @Test public void supportsSubqueriesInComparisons() { if (dbmd != null) { - Assertions.assertThrows(UJESSQLException.class, () -> dbmd.supportsSubqueriesInComparisons()); + Assertions.assertThrows( + LinkisSQLException.class, () -> dbmd.supportsSubqueriesInComparisons()); } } @Test public void getMaxColumnsInSelect() { if (dbmd != null) { - Assertions.assertThrows(UJESSQLException.class, () -> dbmd.getMaxColumnsInSelect()); + Assertions.assertThrows(LinkisSQLException.class, () -> dbmd.getMaxColumnsInSelect()); } } @Test public void nullPlusNonNullIsNull() { if (dbmd != null) { - Assertions.assertThrows(UJESSQLException.class, () -> dbmd.nullPlusNonNullIsNull()); + Assertions.assertThrows(LinkisSQLException.class, () -> dbmd.nullPlusNonNullIsNull()); } } @@ -104,7 +105,7 @@ public void supportsCatalogsInDataManipulation() { public void supportsDataDefinitionAndDataManipulationTransactions() { if (dbmd != null) { Assertions.assertThrows( - UJESSQLException.class, + LinkisSQLException.class, () -> dbmd.supportsDataDefinitionAndDataManipulationTransactions()); } } @@ -112,7 +113,7 @@ public void supportsDataDefinitionAndDataManipulationTransactions() { @Test public void supportsTableCorrelationNames() { if (dbmd != null) { - Assertions.assertThrows(UJESSQLException.class, () -> dbmd.supportsTableCorrelationNames()); + Assertions.assertThrows(LinkisSQLException.class, () -> dbmd.supportsTableCorrelationNames()); } } @@ -133,7 +134,7 @@ public void supportsFullOuterJoins() { @Test public void supportsExpressionsInOrderBy() { if (dbmd != null) { - Assertions.assertThrows(UJESSQLException.class, () -> dbmd.supportsExpressionsInOrderBy()); + Assertions.assertThrows(LinkisSQLException.class, () -> dbmd.supportsExpressionsInOrderBy()); } } @@ -147,14 +148,14 @@ public void allProceduresAreCallable() { @Test public void getMaxTablesInSelect() { if (dbmd != null) { - Assertions.assertThrows(UJESSQLException.class, () -> dbmd.getMaxTablesInSelect()); + Assertions.assertThrows(LinkisSQLException.class, () -> dbmd.getMaxTablesInSelect()); } } @Test public void nullsAreSortedAtStart() { if (dbmd != null) { - Assertions.assertThrows(UJESSQLException.class, () -> dbmd.nullsAreSortedAtStart()); + Assertions.assertThrows(LinkisSQLException.class, () -> dbmd.nullsAreSortedAtStart()); } } @@ -168,7 +169,7 @@ public void supportsPositionedUpdate() { @Test public void ownDeletesAreVisible() { if (dbmd != null) { - Assertions.assertThrows(UJESSQLException.class, () -> dbmd.ownDeletesAreVisible(0)); + Assertions.assertThrows(LinkisSQLException.class, () -> dbmd.ownDeletesAreVisible(0)); } } @@ -182,14 +183,14 @@ public void supportsResultSetHoldability() { @Test public void getMaxStatements() { if (dbmd != null) { - Assertions.assertThrows(UJESSQLException.class, () -> dbmd.getMaxStatements()); + Assertions.assertThrows(LinkisSQLException.class, () -> dbmd.getMaxStatements()); } } @Test public void getRowIdLifetime() { if (dbmd != null) { - Assertions.assertThrows(UJESSQLException.class, () -> dbmd.getRowIdLifetime()); + Assertions.assertThrows(LinkisSQLException.class, () -> dbmd.getRowIdLifetime()); } } diff --git a/linkis-computation-governance/linkis-jdbc-driver/src/test/java/org/apache/linkis/ujes/jdbc/UJESSQLPreparedStatementTest.java b/linkis-computation-governance/linkis-jdbc-driver/src/test/java/org/apache/linkis/ujes/jdbc/UJESSQLPreparedStatementTest.java index 54bc3d4538..da431c82e7 100644 --- a/linkis-computation-governance/linkis-jdbc-driver/src/test/java/org/apache/linkis/ujes/jdbc/UJESSQLPreparedStatementTest.java +++ b/linkis-computation-governance/linkis-jdbc-driver/src/test/java/org/apache/linkis/ujes/jdbc/UJESSQLPreparedStatementTest.java @@ -35,8 +35,8 @@ * */ public class UJESSQLPreparedStatementTest { - private static UJESSQLConnection conn; - private UJESSQLPreparedStatement preStatement; + private static LinkisSQLConnection conn; + private LinkisSQLPreparedStatement preStatement; @BeforeAll public static void getConnection() { diff --git a/linkis-computation-governance/linkis-jdbc-driver/src/test/java/org/apache/linkis/ujes/jdbc/UJESSQLResultSetTest.java b/linkis-computation-governance/linkis-jdbc-driver/src/test/java/org/apache/linkis/ujes/jdbc/UJESSQLResultSetTest.java index 0a545d8012..a8f0a179d0 100644 --- a/linkis-computation-governance/linkis-jdbc-driver/src/test/java/org/apache/linkis/ujes/jdbc/UJESSQLResultSetTest.java +++ b/linkis-computation-governance/linkis-jdbc-driver/src/test/java/org/apache/linkis/ujes/jdbc/UJESSQLResultSetTest.java @@ -33,8 +33,8 @@ public class UJESSQLResultSetTest { - private static UJESSQLConnection conn; - private UJESSQLPreparedStatement preStatement; + private static LinkisSQLConnection conn; + private LinkisSQLPreparedStatement preStatement; private UJESSQLResultSet resultSet; private UJESSQLResultSetMetaData metaData; diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/pom.xml b/linkis-computation-governance/linkis-manager/linkis-application-manager/pom.xml index 99f4613974..289939aa3c 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/pom.xml +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/pom.xml @@ -95,6 +95,12 @@ provided + + org.apache.linkis + linkis-ps-common-lock + ${project.version} + + org.json4s json4s-jackson_${scala.binary.version} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/LinkisManagerApplication.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/LinkisManagerApplication.java similarity index 96% rename from linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/LinkisManagerApplication.java rename to linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/LinkisManagerApplication.java index cfbefdbaaa..064d61a6fb 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/LinkisManagerApplication.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/LinkisManagerApplication.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.linkis.manager.am; +package org.apache.linkis.manager; import org.apache.linkis.LinkisBaseServerApp; diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/event/message/EngineConnPidCallbackEvent.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/event/message/EngineConnPidCallbackEvent.java new file mode 100644 index 0000000000..1202433fb2 --- /dev/null +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/event/message/EngineConnPidCallbackEvent.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.am.event.message; + +import org.apache.linkis.governance.common.protocol.task.ResponseEngineConnPid; + +import org.springframework.context.ApplicationEvent; + +public class EngineConnPidCallbackEvent extends ApplicationEvent { + + private ResponseEngineConnPid protocol; + + public EngineConnPidCallbackEvent(ResponseEngineConnPid protocol) { + super(protocol); + this.protocol = protocol; + } + + public ResponseEngineConnPid getProtocol() { + return protocol; + } +} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/exception/AMErrorCode.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/exception/AMErrorCode.java index 3734e3bdf6..c05768739c 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/exception/AMErrorCode.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/exception/AMErrorCode.java @@ -28,7 +28,11 @@ public enum AMErrorCode implements LinkisErrorCode { NOT_EXISTS_ENGINE_CONN(210003, "Not exists EngineConn(不存在的引擎)"), - AM_CONF_ERROR(210004, "AM configuration error(AM配置错误)"); + AM_CONF_ERROR(210004, "AM configuration error(AM配置错误)"), + + ASK_ENGINE_ERROR_RETRY(210005, "Ask engine error, retry(请求引擎失败，重试)"), + + EC_OPERATE_ERROR(210006, "Failed to execute operation(引擎操作失败)"); private final int errorCode; diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/ECResourceInfoRestfulApi.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/ECResourceInfoRestfulApi.java index 03a2b1465d..5f0544d390 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/ECResourceInfoRestfulApi.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/ECResourceInfoRestfulApi.java @@ -71,9 +71,10 @@ public class ECResourceInfoRestfulApi { public Message getECInfo( HttpServletRequest req, @RequestParam(value = "ticketid") String ticketid) throws AMErrorException { + logger.info("ticked: {} get ec info", ticketid); ECResourceInfoRecord ecResourceInfoRecord = ecResourceInfoService.getECResourceInfoRecord(ticketid); - String userName = ModuleUserUtils.getOperationUser(req, "getECInfo ticketid:") + ticketid; + String userName = ModuleUserUtils.getOperationUser(req, "getECInfo ticketid:" + ticketid); if (null != ecResourceInfoRecord && (userName.equalsIgnoreCase(ecResourceInfoRecord.getCreateUser()) || Configuration.isAdmin(userName))) { @@ -113,6 +114,7 @@ public Message deleteECInfo(HttpServletRequest req, @PathVariable("ticketid") St @ApiImplicitParam(name = "startDate", dataType = "String", value = "start date"), @ApiImplicitParam(name = "endDate", dataType = "String", value = "end date"), @ApiImplicitParam(name = "engineType", dataType = "String", value = "engine type"), + @ApiImplicitParam(name = "status", dataType = "String", value = "engine status"), @ApiImplicitParam(name = "pageNow", dataType = "String", value = "page now"), @ApiImplicitParam(name = "pageSize", dataType = "String", value = "page size") }) @@ -131,6 +133,7 @@ public Message queryEcrHistory( defaultValue = "#{new java.util.Date()}") Date endDate, @RequestParam(value = "engineType", required = false) String engineType, + @RequestParam(value = "status", required = false) String status, @RequestParam(value = "pageNow", required = false, defaultValue = "1") Integer pageNow, @RequestParam(value = "pageSize", required = false, defaultValue = "20") Integer pageSize) { String username = SecurityFilter.getLoginUsername(req); @@ -138,6 +141,7 @@ public Message queryEcrHistory( instance = ECResourceInfoUtils.strCheckAndDef(instance, null); String creatorUser = ECResourceInfoUtils.strCheckAndDef(creator, null); engineType = ECResourceInfoUtils.strCheckAndDef(engineType, null); + status = ECResourceInfoUtils.strCheckAndDef(status, null); if (null != creatorUser && !ECResourceInfoUtils.checkNameValid(creatorUser)) { return Message.error("Invalid creator : " + creatorUser); } @@ -148,7 +152,7 @@ public Message queryEcrHistory( calendar.set(Calendar.SECOND, 0); startDate = calendar.getTime(); } - if (Configuration.isAdmin(username)) { + if (Configuration.isJobHistoryAdmin(username)) { username = null; if (StringUtils.isNotBlank(creatorUser)) { username = creatorUser; @@ -161,12 +165,12 @@ public Message queryEcrHistory( try { queryTasks = ecResourceInfoService.getECResourceInfoRecordList( - instance, endDate, startDate, username, engineType); + instance, endDate, startDate, username, engineType, status); queryTasks.forEach( info -> { ECResourceInfoRecordVo ecrHistroryListVo = new ECResourceInfoRecordVo(); BeanUtils.copyProperties(info, ecrHistroryListVo); - ecrHistroryListVo.setEngineType(info.getLabelValue().split(",")[1].split("-")[0]); + ecrHistroryListVo.setEngineType(info.getEngineType()); ecrHistroryListVo.setUsedResource( ECResourceInfoUtils.getStringToMap(info.getUsedResource(), info)); ecrHistroryListVo.setReleasedResource( @@ -188,25 +192,49 @@ public Message queryEcrHistory( @ApiImplicitParam(name = "creators", dataType = "Array", required = true, value = "creators"), @ApiImplicitParam(name = "engineTypes", dataType = "Array", value = "engine type"), @ApiImplicitParam(name = "statuss", dataType = "Array", value = "statuss"), + @ApiImplicitParam(name = "queueName", dataType = "String", value = "queueName"), + @ApiImplicitParam(name = "ecInstances", dataType = "Array", value = "ecInstances"), }) @RequestMapping(path = "/ecList", method = RequestMethod.POST) public Message queryEcList(HttpServletRequest req, @RequestBody JsonNode jsonNode) { + String username = ModuleUserUtils.getOperationUser(req, "ecList"); + String token = ModuleUserUtils.getToken(req); + // check special admin token + if (StringUtils.isNotBlank(token)) { + if (!Configuration.isAdminToken(token)) { + logger.warn("Token:{} has no permission to query ecList.", token); + return Message.error("Token:" + token + " has no permission to query ecList."); + } + } else if (!Configuration.isAdmin(username)) { + logger.warn("User:{} has no permission to query ecList.", username); + return Message.error("User:" + username + " has no permission to query ecList."); + } JsonNode creatorsParam = jsonNode.get("creators"); JsonNode engineTypesParam = jsonNode.get("engineTypes"); JsonNode statussParam = jsonNode.get("statuss"); + JsonNode queueNameParam = jsonNode.get("queueName"); + JsonNode ecInstancesParam = jsonNode.get("ecInstances"); - if (creatorsParam == null || creatorsParam.isNull() || creatorsParam.size() == 0) { - return Message.error("creators is null in the parameters of the request(请求参数中【creators】为空)"); - } + // if (creatorsParam == null || creatorsParam.isNull() || creatorsParam.size() == 0) { + // return Message.error("creators is null in the parameters of the + // request(请求参数中【creators】为空)"); + // } List creatorUserList = new ArrayList<>(); - try { - creatorUserList = - JsonUtils.jackson() - .readValue(creatorsParam.toString(), new TypeReference>() {}); - } catch (JsonProcessingException e) { - return Message.error("parameters:creators parsing failed(请求参数【creators】解析失败)"); + if (creatorsParam != null && !creatorsParam.isNull()) { + try { + creatorUserList = + JsonUtils.jackson() + .readValue(creatorsParam.toString(), new TypeReference>() {}); + } catch (JsonProcessingException e) { + return Message.error("parameters:creators parsing failed(请求参数【creators】解析失败)"); + } + for (String creatorUser : creatorUserList) { + if (null != creatorUser && !ECResourceInfoUtils.checkNameValid(creatorUser)) { + return Message.error("Invalid creator: " + creatorUser); + } + } } List engineTypeList = new ArrayList<>(); @@ -230,35 +258,37 @@ public Message queryEcList(HttpServletRequest req, @RequestBody JsonNode jsonNod return Message.error("parameters:statuss parsing failed(请求参数【statuss】解析失败)"); } } - - String username = ModuleUserUtils.getOperationUser(req, "ecList"); - - String token = ModuleUserUtils.getToken(req); - // check special admin token - if (StringUtils.isNotBlank(token)) { - if (!Configuration.isAdminToken(token)) { - logger.warn("Token:{} has no permission to query ecList.", token); - return Message.error("Token:" + token + " has no permission to query ecList."); + String queueName = ""; + if (queueNameParam != null && !queueNameParam.isNull()) { + try { + queueName = + JsonUtils.jackson() + .readValue(queueNameParam.toString(), new TypeReference() {}); + } catch (JsonProcessingException e) { + return Message.error("parameters:queueName parsing failed(请求参数【queueName】解析失败)"); } - } else if (!Configuration.isAdmin(username)) { - logger.warn("User:{} has no permission to query ecList.", username); - return Message.error("User:" + username + " has no permission to query ecList."); } - - for (String creatorUser : creatorUserList) { - if (null != creatorUser && !ECResourceInfoUtils.checkNameValid(creatorUser)) { - return Message.error("Invalid creator: " + creatorUser); + List ecInstancesList = new ArrayList<>(); + if (ecInstancesParam != null && !ecInstancesParam.isNull()) { + try { + ecInstancesList = + JsonUtils.jackson() + .readValue(ecInstancesParam.toString(), new TypeReference>() {}); + } catch (JsonProcessingException e) { + return Message.error("parameters:instanceName parsing failed(请求参数【ecInstances】解析失败)"); } } - logger.info( - "request parameters creatorUserList:[{}], engineTypeList:[{}], statusStrList:[{}]", + "request parameters creatorUserList:[{}], engineTypeList:[{}], statusStrList:[{}], queueName:{}, instanceNameList:{}", String.join(",", creatorUserList), String.join(",", engineTypeList), - String.join(",", statusStrList)); + String.join(",", statusStrList), + String.join(",", ecInstancesList), + queueNameParam); List> list = - ecResourceInfoService.getECResourceInfoList(creatorUserList, engineTypeList, statusStrList); + ecResourceInfoService.getECResourceInfoList( + creatorUserList, engineTypeList, statusStrList, queueName, ecInstancesList); return Message.ok().data("ecList", list); } diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/EMRestfulApi.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/EMRestfulApi.java index 4d8bbad27d..d21a6631a4 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/EMRestfulApi.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/EMRestfulApi.java @@ -25,6 +25,7 @@ import org.apache.linkis.manager.am.exception.AMErrorCode; import org.apache.linkis.manager.am.exception.AMErrorException; import org.apache.linkis.manager.am.manager.EngineNodeManager; +import org.apache.linkis.manager.am.service.ECResourceInfoService; import org.apache.linkis.manager.am.service.em.ECMOperateService; import org.apache.linkis.manager.am.service.em.EMInfoService; import org.apache.linkis.manager.am.utils.AMUtils; @@ -33,6 +34,7 @@ import org.apache.linkis.manager.common.entity.metrics.NodeHealthyInfo; import org.apache.linkis.manager.common.entity.node.EMNode; import org.apache.linkis.manager.common.entity.node.EngineNode; +import org.apache.linkis.manager.common.entity.persistence.ECResourceInfoRecord; import org.apache.linkis.manager.common.protocol.OperateRequest$; import org.apache.linkis.manager.common.protocol.em.ECMOperateRequest; import org.apache.linkis.manager.common.protocol.em.ECMOperateRequest$; @@ -91,6 +93,7 @@ public class EMRestfulApi { @Autowired private ECMOperateService ecmOperateService; + @Autowired private ECResourceInfoService ecResourceInfoService; private LabelBuilderFactory stdLabelBuilderFactory = LabelBuilderFactoryContext.getLabelBuilderFactory(); @@ -323,7 +326,10 @@ public Message executeECMOperationByEC(HttpServletRequest req, @RequestBody Json return Message.error( "You have no permission to execute ECM Operation by this EngineConn " + serviceInstance); } - return executeECMOperation(engineNode.getEMNode(), new ECMOperateRequest(userName, parameters)); + return executeECMOperation( + engineNode.getEMNode(), + engineNode.getServiceInstance().getInstance(), + new ECMOperateRequest(userName, parameters)); } @ApiOperation( @@ -354,7 +360,14 @@ public Message executeECMOperation(HttpServletRequest req, @RequestBody JsonNode "Fail to process the operation parameters, cased by " + ExceptionUtils.getRootCauseMessage(e)); } - return executeECMOperation(ecmNode, new ECMOperateRequest(userName, parameters)); + if (parameters.containsKey("engineConnInstance")) { + return executeECMOperation( + ecmNode, + parameters.get("engineConnInstance").toString(), + new ECMOperateRequest(userName, parameters)); + } else { + return executeECMOperation(ecmNode, "", new ECMOperateRequest(userName, parameters)); + } } @ApiOperation(value = "openEngineLog", notes = "open Engine log", response = Message.class) @@ -377,9 +390,10 @@ public Message openEngineLog(HttpServletRequest req, @RequestBody JsonNode jsonN String userName = ModuleUserUtils.getOperationUser(req, "openEngineLog"); EMNode ecmNode; Map parameters; + String engineInstance; try { String emInstance = jsonNode.get("emInstance").asText(); - String engineInstance = jsonNode.get("instance").asText(); + engineInstance = jsonNode.get("instance").asText(); ServiceInstance serviceInstance = EngineRestfulApi.getServiceInstance(jsonNode); logger.info("User {} try to open engine: {} log.", userName, serviceInstance); ecmNode = @@ -392,7 +406,10 @@ public Message openEngineLog(HttpServletRequest req, @RequestBody JsonNode jsonN jsonNode.get("parameters").toString(), new TypeReference>() {}); String logType = (String) parameters.get("logType"); - if (!logType.equals("stdout") && !logType.equals("stderr")) { + if (!logType.equals("stdout") + && !logType.equals("stderr") + && !logType.equals("gc") + && !logType.equals("yarnApp")) { throw new AMErrorException( AMErrorCode.PARAM_ERROR.getErrorCode(), AMErrorCode.PARAM_ERROR.getErrorDesc()); } @@ -413,16 +430,18 @@ public Message openEngineLog(HttpServletRequest req, @RequestBody JsonNode jsonN logger.error("Failed to open engine log, error:", e); return Message.error(e.getMessage()); } - return executeECMOperation(ecmNode, new ECMOperateRequest(userName, parameters)); + return executeECMOperation( + ecmNode, engineInstance, new ECMOperateRequest(userName, parameters)); } - private Message executeECMOperation(EMNode ecmNode, ECMOperateRequest ecmOperateRequest) { + private Message executeECMOperation( + EMNode ecmNode, String engineInstance, ECMOperateRequest ecmOperateRequest) { String operationName = OperateRequest$.MODULE$.getOperationName(ecmOperateRequest.parameters()); - if (ArrayUtils.contains(adminOperations, operationName) - && Configuration.isNotAdmin(ecmOperateRequest.user())) { + String userName = ecmOperateRequest.user(); + if (ArrayUtils.contains(adminOperations, operationName) && Configuration.isNotAdmin(userName)) { logger.warn( "User {} has no permission to execute {} admin Operation in ECM {}.", - ecmOperateRequest.user(), + userName, operationName, ecmNode.getServiceInstance()); return Message.error( @@ -431,6 +450,33 @@ private Message executeECMOperation(EMNode ecmNode, ECMOperateRequest ecmOperate + " admin Operation in ECM " + ecmNode.getServiceInstance()); } + + // fill in logDirSuffix + if (StringUtils.isNotBlank(engineInstance) + && Objects.isNull(ecmOperateRequest.parameters().get("logDirSuffix"))) { + ECResourceInfoRecord ecResourceInfoRecord = + ecResourceInfoService.getECResourceInfoRecordByInstance(engineInstance); + if (Objects.isNull(ecResourceInfoRecord)) { + return Message.error("ECM instance: " + ecmNode.getServiceInstance() + " not exist "); + } + // eg logDirSuffix -> root/20230705/io_file/6d48068a-0e1e-44b5-8eb2-835034db5b30/logs + String logDirSuffix = ecResourceInfoRecord.getLogDirSuffix(); + if (!userName.equals(ecResourceInfoRecord.getCreateUser()) + && Configuration.isNotJobHistoryAdmin(userName)) { + logger.warn( + "User {} has no permission to get log with path: {} in ECM:{}.", + userName, + logDirSuffix, + ecmNode.getServiceInstance()); + return Message.error( + "You have no permission to get log with path:" + + logDirSuffix + + " in ECM:" + + ecmNode.getServiceInstance()); + } + ecmOperateRequest.parameters().put("logDirSuffix", logDirSuffix); + } + ECMOperateResponse engineOperateResponse = ecmOperateService.executeOperation(ecmNode, ecmOperateRequest); diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/EngineRestfulApi.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/EngineRestfulApi.java index 475c58c9e8..70b74387ca 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/EngineRestfulApi.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/EngineRestfulApi.java @@ -23,28 +23,28 @@ import org.apache.linkis.common.utils.ByteTimeUtils; import org.apache.linkis.common.utils.JsonUtils; import org.apache.linkis.governance.common.conf.GovernanceCommonConf; +import org.apache.linkis.governance.common.constant.ec.ECConstants; +import org.apache.linkis.governance.common.utils.JobUtils; +import org.apache.linkis.governance.common.utils.LoggerUtils; import org.apache.linkis.manager.am.conf.AMConfiguration; import org.apache.linkis.manager.am.exception.AMErrorCode; import org.apache.linkis.manager.am.exception.AMErrorException; import org.apache.linkis.manager.am.manager.EngineNodeManager; import org.apache.linkis.manager.am.service.ECResourceInfoService; -import org.apache.linkis.manager.am.service.engine.EngineCreateService; -import org.apache.linkis.manager.am.service.engine.EngineInfoService; -import org.apache.linkis.manager.am.service.engine.EngineOperateService; -import org.apache.linkis.manager.am.service.engine.EngineStopService; +import org.apache.linkis.manager.am.service.engine.*; import org.apache.linkis.manager.am.util.ECResourceInfoUtils; import org.apache.linkis.manager.am.utils.AMUtils; import org.apache.linkis.manager.am.vo.AMEngineNodeVo; +import org.apache.linkis.manager.common.constant.AMConstant; import org.apache.linkis.manager.common.entity.enumeration.NodeStatus; import org.apache.linkis.manager.common.entity.node.AMEMNode; +import org.apache.linkis.manager.common.entity.node.EMNode; import org.apache.linkis.manager.common.entity.node.EngineNode; import org.apache.linkis.manager.common.entity.persistence.ECResourceInfoRecord; -import org.apache.linkis.manager.common.protocol.engine.EngineCreateRequest; -import org.apache.linkis.manager.common.protocol.engine.EngineOperateRequest; -import org.apache.linkis.manager.common.protocol.engine.EngineOperateResponse; -import org.apache.linkis.manager.common.protocol.engine.EngineStopRequest; +import org.apache.linkis.manager.common.protocol.engine.*; import org.apache.linkis.manager.label.builder.factory.LabelBuilderFactory; import org.apache.linkis.manager.label.builder.factory.LabelBuilderFactoryContext; +import org.apache.linkis.manager.label.constant.LabelKeyConstant; import org.apache.linkis.manager.label.entity.Label; import org.apache.linkis.manager.label.entity.UserModifiable; import org.apache.linkis.manager.label.exception.LabelErrorException; @@ -65,11 +65,10 @@ import java.io.IOException; import java.text.MessageFormat; import java.util.*; +import java.util.concurrent.Callable; import java.util.stream.Collectors; import java.util.stream.Stream; -import scala.annotation.meta.param; - import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JsonNode; @@ -91,6 +90,7 @@ public class EngineRestfulApi { @Autowired private EngineInfoService engineInfoService; + @Autowired private EngineAskEngineService engineAskService; @Autowired private EngineCreateService engineCreateService; @Autowired private EngineNodeManager engineNodeManager; @@ -103,6 +103,8 @@ public class EngineRestfulApi { @Autowired private ECResourceInfoService ecResourceInfoService; + @Autowired private EngineReuseService engineReuseService; + private final ObjectMapper objectMapper = new ObjectMapper(); private LabelBuilderFactory stdLabelBuilderFactory = @@ -110,6 +112,183 @@ public class EngineRestfulApi { private static final Logger logger = LoggerFactory.getLogger(EngineRestfulApi.class); + @ApiOperation(value = "askEngineConn", response = Message.class) + @ApiOperationSupport(ignoreParameters = {"jsonNode"}) + @RequestMapping(path = "/askEngineConn", method = RequestMethod.POST) + public Message askEngineConn( + HttpServletRequest req, @RequestBody EngineAskRequest engineAskRequest) + throws IOException, InterruptedException { + String userName = ModuleUserUtils.getOperationUser(req, "askEngineConn"); + engineAskRequest.setUser(userName); + long timeout = engineAskRequest.getTimeOut(); + if (timeout <= 0) { + timeout = AMConfiguration.ENGINE_CONN_START_REST_MAX_WAIT_TIME().getValue().toLong(); + engineAskRequest.setTimeOut(timeout); + } + Map retEngineNode = new HashMap<>(); + logger.info( + "User {} try to ask an engineConn with maxStartTime {}. EngineAskRequest is {}.", + userName, + ByteTimeUtils.msDurationToString(timeout), + engineAskRequest); + Sender sender = Sender.getSender(Sender.getThisServiceInstance()); + EngineNode engineNode = null; + + // try to reuse ec first + String taskId = JobUtils.getJobIdFromStringMap(engineAskRequest.getProperties()); + LoggerUtils.setJobIdMDC(taskId); + logger.info("received task : {}, engineAskRequest : {}", taskId, engineAskRequest); + if (!engineAskRequest.getLabels().containsKey(LabelKeyConstant.EXECUTE_ONCE_KEY)) { + EngineReuseRequest engineReuseRequest = new EngineReuseRequest(); + engineReuseRequest.setLabels(engineAskRequest.getLabels()); + engineReuseRequest.setTimeOut(engineAskRequest.getTimeOut()); + engineReuseRequest.setUser(engineAskRequest.getUser()); + engineReuseRequest.setProperties(engineAskRequest.getProperties()); + boolean end = false; + EngineNode reuseNode = null; + int count = 0; + int MAX_RETRY = 2; + while (!end) { + try { + reuseNode = engineReuseService.reuseEngine(engineReuseRequest, sender); + end = true; + } catch (LinkisRetryException e) { + logger.error( + "task: {}, user: {} reuse engine failed", taskId, engineReuseRequest.getUser(), e); + Thread.sleep(1000); + end = false; + count += 1; + if (count > MAX_RETRY) { + end = true; + } + } catch (Exception e1) { + logger.info( + "task: {} user: {} reuse engine failed", taskId, engineReuseRequest.getUser(), e1); + end = true; + } + } + if (null != reuseNode) { + logger.info( + "Finished to ask engine for task: {}, user: {} by reuse node {}", + taskId, + engineReuseRequest.getUser(), + reuseNode); + LoggerUtils.removeJobIdMDC(); + engineNode = reuseNode; + } + } + + if (null != engineNode) { + fillResultEngineNode(retEngineNode, engineNode); + return Message.ok("reuse engineConn ended.").data("engine", retEngineNode); + } + + String engineAskAsyncId = EngineAskEngineService$.MODULE$.getAsyncId(); + Callable createECTask = + new Callable() { + @Override + public Object call() { + LoggerUtils.setJobIdMDC(taskId); + logger.info( + "Task: {}, start to async({}) createEngine: {}", + taskId, + engineAskAsyncId, + engineAskRequest.getCreateService()); + // remove engineInstance label if exists + engineAskRequest.getLabels().remove("engineInstance"); + EngineCreateRequest engineCreateRequest = new EngineCreateRequest(); + engineCreateRequest.setLabels(engineAskRequest.getLabels()); + engineCreateRequest.setTimeout(engineAskRequest.getTimeOut()); + engineCreateRequest.setUser(engineAskRequest.getUser()); + engineCreateRequest.setProperties(engineAskRequest.getProperties()); + engineCreateRequest.setCreateService(engineAskRequest.getCreateService()); + try { + EngineNode createNode = engineCreateService.createEngine(engineCreateRequest, sender); + long timeout = 0L; + if (engineCreateRequest.getTimeout() <= 0) { + timeout = AMConfiguration.ENGINE_START_MAX_TIME().getValue().toLong(); + } else { + timeout = engineCreateRequest.getTimeout(); + } + // useEngine need to add timeout + EngineNode createEngineNode = engineNodeManager.useEngine(createNode, timeout); + if (null == createEngineNode) { + throw new LinkisRetryException( + AMConstant.EM_ERROR_CODE, + "create engine${createNode.getServiceInstance} success, but to use engine failed"); + } + logger.info( + "Task: $taskId finished to ask engine for user ${engineAskRequest.getUser} by create node $createEngineNode"); + return createEngineNode; + } catch (Exception e) { + logger.error( + "Task: {} failed to ask engine for user {} by create node", taskId, userName, e); + return new LinkisRetryException(AMConstant.EM_ERROR_CODE, e.getMessage()); + } finally { + LoggerUtils.removeJobIdMDC(); + } + } + }; + + try { + Object rs = createECTask.call(); + if (rs instanceof LinkisRetryException) { + throw (LinkisRetryException) rs; + } else { + engineNode = (EngineNode) rs; + } + } catch (LinkisRetryException retryException) { + logger.error( + "User {} create engineConn failed get retry exception. can be Retry", + userName, + retryException); + return Message.error( + String.format( + "Create engineConn failed, caused by %s.", + ExceptionUtils.getRootCauseMessage(retryException))) + .data("canRetry", true); + } catch (Exception e) { + LoggerUtils.removeJobIdMDC(); + logger.error("User {} create engineConn failed get retry exception", userName, e); + return Message.error( + String.format( + "Create engineConn failed, caused by %s.", ExceptionUtils.getRootCauseMessage(e))); + } + + LoggerUtils.removeJobIdMDC(); + fillResultEngineNode(retEngineNode, engineNode); + logger.info( + "Finished to create a engineConn for user {}. NodeInfo is {}.", userName, engineNode); + // to transform to a map + return Message.ok("create engineConn ended.").data("engine", retEngineNode); + } + + private void fillNullNode( + Map retEngineNode, EngineAskAsyncResponse askAsyncResponse) { + retEngineNode.put(AMConstant.EC_ASYNC_START_RESULT_KEY, AMConstant.EC_ASYNC_START_RESULT_FAIL); + retEngineNode.put( + AMConstant.EC_ASYNC_START_FAIL_MSG_KEY, + "Got null response for asyId : " + askAsyncResponse.id()); + retEngineNode.put(ECConstants.MANAGER_SERVICE_INSTANCE_KEY(), Sender.getThisServiceInstance()); + } + + private void fillResultEngineNode(Map retEngineNode, EngineNode engineNode) { + retEngineNode.put( + AMConstant.EC_ASYNC_START_RESULT_KEY, AMConstant.EC_ASYNC_START_RESULT_SUCCESS); + retEngineNode.put("serviceInstance", engineNode.getServiceInstance()); + if (null == engineNode.getNodeStatus()) { + engineNode.setNodeStatus(NodeStatus.Starting); + } + retEngineNode.put(ECConstants.NODE_STATUS_KEY(), engineNode.getNodeStatus().toString()); + retEngineNode.put(ECConstants.EC_TICKET_ID_KEY(), engineNode.getTicketId()); + EMNode emNode = engineNode.getEMNode(); + if (null != emNode) { + retEngineNode.put( + ECConstants.ECM_SERVICE_INSTANCE_KEY(), engineNode.getEMNode().getServiceInstance()); + } + retEngineNode.put(ECConstants.MANAGER_SERVICE_INSTANCE_KEY(), Sender.getThisServiceInstance()); + } + @ApiOperation(value = "createEngineConn", response = Message.class) @ApiOperationSupport(ignoreParameters = {"jsonNode"}) @RequestMapping(path = "/createEngineConn", method = RequestMethod.POST) @@ -149,13 +328,7 @@ public Message createEngineConn( "Finished to create a engineConn for user {}. NodeInfo is {}.", userName, engineNode); // to transform to a map Map retEngineNode = new HashMap<>(); - retEngineNode.put("serviceInstance", engineNode.getServiceInstance()); - if (null == engineNode.getNodeStatus()) { - engineNode.setNodeStatus(NodeStatus.Starting); - } - retEngineNode.put("nodeStatus", engineNode.getNodeStatus().toString()); - retEngineNode.put("ticketId", engineNode.getTicketId()); - retEngineNode.put("ecmServiceInstance", engineNode.getEMNode().getServiceInstance()); + fillResultEngineNode(retEngineNode, engineNode); return Message.ok("create engineConn succeed.").data("engine", retEngineNode); } @@ -173,6 +346,7 @@ public Message getEngineConn(HttpServletRequest req, @RequestBody JsonNode jsonN } catch (Exception e) { logger.info("Instances {} does not exist", serviceInstance.getInstance()); } + String ecMetrics = null; if (null == engineNode) { ECResourceInfoRecord ecInfo = null; if (null != ticketIdNode) { @@ -189,12 +363,19 @@ public Message getEngineConn(HttpServletRequest req, @RequestBody JsonNode jsonN if (null == ecInfo) { return Message.error("Instance does not exist " + serviceInstance); } + if (null == ecMetrics) { + ecMetrics = ecInfo.getMetrics(); + } engineNode = ECResourceInfoUtils.convertECInfoTOECNode(ecInfo); + } else { + ecMetrics = engineNode.getEcMetrics(); } if (!userName.equals(engineNode.getOwner()) && Configuration.isNotAdmin(userName)) { return Message.error("You have no permission to access EngineConn " + serviceInstance); } - return Message.ok().data("engine", engineNode); + Message result = Message.ok().data("engine", engineNode); + result.data(AMConstant.EC_METRICS_KEY, ecMetrics); + return result; } @ApiOperation(value = "kill egineconn", notes = "kill engineconn", response = Message.class) @@ -484,6 +665,11 @@ public Message executeEngineConnOperation(HttpServletRequest req, @RequestBody J ServiceInstance serviceInstance = getServiceInstance(jsonNode); logger.info("User {} try to execute Engine Operation {}.", userName, serviceInstance); EngineNode engineNode = engineNodeManager.getEngineNode(serviceInstance); + if (null == engineNode) { + return Message.ok() + .data("isError", true) + .data("errorMsg", "Ec : " + serviceInstance.toString() + " not found."); + } if (!userName.equals(engineNode.getOwner()) && Configuration.isNotAdmin(userName)) { return Message.error("You have no permission to execute Engine Operation " + serviceInstance); } diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/ECResourceInfoService.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/ECResourceInfoService.java index 6aabe19198..06299cf502 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/ECResourceInfoService.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/ECResourceInfoService.java @@ -34,14 +34,25 @@ public interface ECResourceInfoService { void deleteECResourceInfoRecord(Integer id); List getECResourceInfoRecordList( - String instance, Date endDate, Date startDate, String username, String engineType); + String instance, + Date endDate, + Date startDate, + String username, + String engineType, + String status); /** * @param creatorUserList engineconn creator list * @param engineTypeList engineconn type list * @param statusStrList engineconn status string list + * @param queueName + * @param ecInstancesList * @return */ List> getECResourceInfoList( - List creatorUserList, List engineTypeList, List statusStrList); + List creatorUserList, + List engineTypeList, + List statusStrList, + String queueName, + List ecInstancesList); } diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/EngineConnPidCallbackService.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/EngineConnPidCallbackService.java similarity index 82% rename from linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/EngineConnPidCallbackService.scala rename to linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/EngineConnPidCallbackService.java index 8e83394738..194eea590d 100644 --- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/EngineConnPidCallbackService.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/EngineConnPidCallbackService.java @@ -15,12 +15,11 @@ * limitations under the License. */ -package org.apache.linkis.ecm.server.service +package org.apache.linkis.manager.am.service; -import org.apache.linkis.governance.common.protocol.task.ResponseEngineConnPid - -trait EngineConnPidCallbackService { - - def dealPid(protocol: ResponseEngineConnPid): Unit +import org.apache.linkis.governance.common.protocol.task.ResponseEngineConnPid; +@FunctionalInterface +public interface EngineConnPidCallbackService { + void dealPid(ResponseEngineConnPid protocol); } diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/EngineConnStatusCallbackService.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/EngineConnStatusCallbackService.java similarity index 79% rename from linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/EngineConnStatusCallbackService.scala rename to linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/EngineConnStatusCallbackService.java index 12ccc088be..477d49aa10 100644 --- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/EngineConnStatusCallbackService.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/EngineConnStatusCallbackService.java @@ -15,12 +15,11 @@ * limitations under the License. */ -package org.apache.linkis.ecm.server.service +package org.apache.linkis.manager.am.service; -import org.apache.linkis.manager.common.protocol.engine.EngineConnStatusCallback +import org.apache.linkis.manager.common.protocol.engine.EngineConnStatusCallbackToAM; -trait EngineConnStatusCallbackService { - - def dealEngineConnStatusCallback(protocol: EngineConnStatusCallback): Unit +public interface EngineConnStatusCallbackService { + void dealEngineConnStatusCallbackToAM(EngineConnStatusCallbackToAM engineConnStatusCallbackToAM); } diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/impl/DefaultEngineConnPidCallbackService.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/impl/DefaultEngineConnPidCallbackService.java new file mode 100644 index 0000000000..3d199fe29c --- /dev/null +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/impl/DefaultEngineConnPidCallbackService.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.am.service.impl; + +import org.apache.linkis.governance.common.protocol.task.ResponseEngineConnPid; +import org.apache.linkis.manager.am.manager.DefaultEngineNodeManager; +import org.apache.linkis.manager.am.service.EngineConnPidCallbackService; +import org.apache.linkis.manager.common.entity.node.EngineNode; +import org.apache.linkis.rpc.message.annotation.Receiver; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Service +public class DefaultEngineConnPidCallbackService implements EngineConnPidCallbackService { + private static final Logger logger = + LoggerFactory.getLogger(DefaultEngineConnPidCallbackService.class); + + @Autowired private DefaultEngineNodeManager defaultEngineNodeManager; + + @Receiver + @Override + public void dealPid(ResponseEngineConnPid protocol) { + // set pid + logger.info( + "DefaultEngineConnPidCallbackService dealPid serviceInstance: [{}] pid: [{}]" + + " ticketId: [{}]", + protocol.serviceInstance(), + protocol.pid(), + protocol.ticketId()); + + EngineNode engineNode = defaultEngineNodeManager.getEngineNode(protocol.serviceInstance()); + if (engineNode == null) { + logger.error( + "DefaultEngineConnPidCallbackService dealPid failed, engineNode is null, serviceInstance:{}", + protocol.serviceInstance()); + return; + } + + engineNode.setIdentifier(protocol.pid()); + defaultEngineNodeManager.updateEngine(engineNode); + } +} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/impl/DefaultEngineConnStatusCallbackService.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/impl/DefaultEngineConnStatusCallbackService.java new file mode 100644 index 0000000000..7efeabd3e8 --- /dev/null +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/impl/DefaultEngineConnStatusCallbackService.java @@ -0,0 +1,133 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.am.service.impl; + +import org.apache.linkis.manager.am.conf.AMConfiguration; +import org.apache.linkis.manager.am.service.EngineConnStatusCallbackService; +import org.apache.linkis.manager.am.service.engine.EngineStopService; +import org.apache.linkis.manager.common.constant.AMConstant; +import org.apache.linkis.manager.common.entity.enumeration.NodeStatus; +import org.apache.linkis.manager.common.entity.metrics.AMNodeMetrics; +import org.apache.linkis.manager.common.protocol.engine.EngineConnStatusCallback; +import org.apache.linkis.manager.common.protocol.engine.EngineConnStatusCallbackToAM; +import org.apache.linkis.manager.persistence.NodeMetricManagerPersistence; +import org.apache.linkis.manager.service.common.metrics.MetricsConverter; +import org.apache.linkis.rpc.message.annotation.Receiver; +import org.apache.linkis.server.BDPJettyServerHelper; + +import org.apache.commons.lang3.StringUtils; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.io.UnsupportedEncodingException; +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; + +import com.fasterxml.jackson.core.JsonProcessingException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Service +public class DefaultEngineConnStatusCallbackService implements EngineConnStatusCallbackService { + + private static final Logger logger = + LoggerFactory.getLogger(DefaultEngineConnStatusCallbackService.class); + + @Autowired private NodeMetricManagerPersistence nodeMetricManagerPersistence; + + @Autowired private MetricsConverter metricsConverter; + + @Autowired private EngineStopService engineStopService; + + private static final String[] canRetryLogs = + AMConfiguration.AM_CAN_RETRY_LOGS().getValue().split(";"); + + @Receiver + public void dealEngineConnStatusCallback(EngineConnStatusCallback protocol) { + logger.info( + "EngineConnStatusCallbackServiceImpl handle engineConnStatus callback serviceInstance: [{}] status: [{}]", + protocol.serviceInstance(), + protocol.status()); + if (!NodeStatus.isAvailable(protocol.status())) { + dealEngineConnStatusCallbackToAM( + new EngineConnStatusCallbackToAM( + protocol.serviceInstance(), protocol.status(), protocol.initErrorMsg(), false)); + } + } + + @Receiver + public void dealEngineConnStatusCallbackToAM( + EngineConnStatusCallbackToAM engineConnStatusCallbackToAM) { + if (engineConnStatusCallbackToAM.serviceInstance() == null) { + logger.warn("call back service instance is null"); + } + logger.info( + "EngineConnStatusCallbackServiceImpl start to deal engineConnStatusCallbackToAM {}", + engineConnStatusCallbackToAM); + + AMNodeMetrics nodeMetrics = new AMNodeMetrics(); + Map heartBeatMsg = new HashMap<>(); + int initErrorMsgMaxByteNum = 60000; + + String initErrorMsg = engineConnStatusCallbackToAM.initErrorMsg(); + try { + if (StringUtils.isNotBlank(initErrorMsg) + && initErrorMsg.getBytes("utf-8").length >= initErrorMsgMaxByteNum) { + initErrorMsg = initErrorMsg.substring(0, initErrorMsgMaxByteNum); + } + } catch (UnsupportedEncodingException e) { + logger.warn("dealEngineConnStatusCallbackToAM getBytes failed", e); + } + heartBeatMsg.put(AMConstant.START_REASON, initErrorMsg); + + if (engineConnStatusCallbackToAM.canRetry()) { + heartBeatMsg.put(AMConstant.EC_CAN_RETRY, engineConnStatusCallbackToAM.canRetry()); + } else if (matchRetryLog(engineConnStatusCallbackToAM.initErrorMsg())) { + logger.info("match canRetry log {}", engineConnStatusCallbackToAM.serviceInstance()); + heartBeatMsg.put(AMConstant.EC_CAN_RETRY, engineConnStatusCallbackToAM.canRetry()); + } + + try { + nodeMetrics.setHeartBeatMsg( + BDPJettyServerHelper.jacksonJson().writeValueAsString(heartBeatMsg)); + } catch (JsonProcessingException e) { + logger.warn("dealEngineConnStatusCallbackToAM writeValueAsString failed", e); + } + nodeMetrics.setServiceInstance(engineConnStatusCallbackToAM.serviceInstance()); + nodeMetrics.setStatus(metricsConverter.convertStatus(engineConnStatusCallbackToAM.status())); + + nodeMetricManagerPersistence.addOrupdateNodeMetrics(nodeMetrics); + logger.info("Finished to deal engineConnStatusCallbackToAM {}", engineConnStatusCallbackToAM); + } + + private boolean matchRetryLog(String errorMsg) { + boolean flag = false; + if (StringUtils.isNotBlank(errorMsg)) { + String errorMsgLowCase = errorMsg.toLowerCase(Locale.getDefault()); + for (String canRetry : canRetryLogs) { + if (errorMsgLowCase.contains(canRetry)) { + logger.info("match engineConn log fatal logs, is {}", canRetry); + flag = true; + } + } + } + return flag; + } +} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/impl/ECResourceInfoServiceImpl.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/impl/ECResourceInfoServiceImpl.java index e8988fee9b..94d7207714 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/impl/ECResourceInfoServiceImpl.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/impl/ECResourceInfoServiceImpl.java @@ -28,12 +28,15 @@ import org.apache.linkis.manager.dao.NodeManagerMapper; import org.apache.linkis.manager.label.service.NodeLabelService; import org.apache.linkis.manager.persistence.LabelManagerPersistence; +import org.apache.linkis.server.BDPJettyServerHelper; +import org.apache.commons.collections.MapUtils; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; +import java.math.BigDecimal; import java.text.SimpleDateFormat; import java.util.*; import java.util.stream.Collectors; @@ -94,14 +97,23 @@ public void deleteECResourceInfoRecord(Integer id) { @Override public List getECResourceInfoRecordList( - String instance, Date endDate, Date startDate, String username, String engineType) { + String instance, + Date endDate, + Date startDate, + String username, + String engineType, + String status) { return ecResourceRecordMapper.getECResourceInfoHistory( - username, instance, endDate, startDate, engineType); + username, instance, endDate, startDate, engineType, status); } @Override public List> getECResourceInfoList( - List creatorUserList, List engineTypeList, List statusStrList) { + List creatorUserList, + List engineTypeList, + List statusStrList, + String queueName, + List ecInstancesList) { List> resultList = new ArrayList<>(); @@ -113,7 +125,7 @@ public List> getECResourceInfoList( // get engine conn info list filter by creator user list /instance status list List ecNodesInfo = - nodeManagerMapper.getEMNodeInfoList(creatorUserList, statusIntList); + nodeManagerMapper.getEMNodeInfoList(creatorUserList, statusIntList, ecInstancesList); // map k:v---> instanceName：PersistencerEcNodeInfo Map persistencerEcNodeInfoMap = @@ -143,8 +155,8 @@ public List> getECResourceInfoList( json.writeValueAsString(ecNodeinfo), new TypeReference>() {}); - Integer intStatus = ecNodeinfo.getInstanceStatus(); - item.put("instanceStatus", NodeStatus.values()[intStatus].name()); + Integer instanceStatus = ecNodeinfo.getInstanceStatus(); + item.put("instanceStatus", NodeStatus.values()[instanceStatus].name()); String usedResourceStr = latestRecord.getUsedResource(); /* @@ -152,12 +164,37 @@ public List> getECResourceInfoList( -> {"driver":{"instance":1,"memory":"2.0 GB","cpu":1} } */ - + long lastUnlockTimestamp = 0L; + if (NodeStatus.values()[instanceStatus].name().equals(NodeStatus.Unlock.name())) { + String heartbeatMsg = ecNodeinfo.getHeartbeatMsg(); + Map heartbeatMap = new HashMap<>(); + if (StringUtils.isNotBlank(heartbeatMsg)) { + heartbeatMap = + BDPJettyServerHelper.gson() + .fromJson(heartbeatMsg, new HashMap<>().getClass()); + } + Object lastUnlockTimestampObject = + heartbeatMap.getOrDefault("lastUnlockTimestamp", 0); + BigDecimal lastUnlockTimestampBigDecimal = + new BigDecimal(String.valueOf(lastUnlockTimestampObject)); + lastUnlockTimestamp = lastUnlockTimestampBigDecimal.longValue(); + } + item.put("lastUnlockTimestamp", lastUnlockTimestamp); item.put("useResource", ECResourceInfoUtils.getStringToMap(usedResourceStr)); item.put("ecmInstance", latestRecord.getEcmInstance()); - String engineType = latestRecord.getLabelValue().split(",")[1].split("-")[0]; + String engineType = latestRecord.getEngineType(); item.put("engineType", engineType); - resultList.add(item); + if (StringUtils.isNotBlank(queueName)) { + Map usedResourceMap = + ECResourceInfoUtils.getStringToMap(usedResourceStr); + Map yarn = MapUtils.getMap(usedResourceMap, "yarn", new HashMap()); + String queueNameStr = String.valueOf(yarn.getOrDefault("queueName", "")); + if (StringUtils.isNotBlank(queueNameStr) && queueName.equals(queueNameStr)) { + resultList.add(item); + } + } else { + resultList.add(item); + } } catch (JsonProcessingException e) { logger.error("Fail to process the ec node info: [{}]", ecNodeinfo, e); } diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/util/ECResourceInfoUtils.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/util/ECResourceInfoUtils.java index ab926df64e..85c7470ce5 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/util/ECResourceInfoUtils.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/util/ECResourceInfoUtils.java @@ -61,7 +61,7 @@ public static ResourceVo getStringToMap(String str, ECResourceInfoRecord info) { Map divermap = MapUtils.getMap(map, "driver"); resourceVo.setInstance(((Double) divermap.get("instance")).intValue()); resourceVo.setCores(((Double) divermap.get("cpu")).intValue()); - String memoryStr = String.valueOf(map.getOrDefault("memory", "0k")); + String memoryStr = String.valueOf(divermap.getOrDefault("memory", "0k")); long memorylong = 0; if (!getScientific(memoryStr)) { memorylong = ByteTimeUtils.byteStringAsBytes(memoryStr); @@ -125,6 +125,7 @@ public static AMEngineNode convertECInfoTOECNode(ECResourceInfoRecord ecInfo) { engineNode.setTicketId(ecInfo.getTicketId()); engineNode.setStartTime(ecInfo.getCreateTime()); engineNode.setUpdateTime(ecInfo.getReleaseTime()); + engineNode.setEcMetrics(ecInfo.getMetrics()); return engineNode; } diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/exception/RMErrorCode.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/exception/RMErrorCode.java index b513931414..eb69062d89 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/exception/RMErrorCode.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/exception/RMErrorCode.java @@ -61,6 +61,8 @@ public enum RMErrorCode implements LinkisErrorCode { CLUSTER_QUEUE_INSTANCES_INSUFFICIENT(12012, "Insufficient cluster queue instance(集群队列实例不足)"), + ACROSS_CLUSTER_RULE_FAILED(12012, "across cluster rule failed(跨集群规则失败)"), + ECM_RESOURCE_INSUFFICIENT(11000, "ECM resources are insufficient(ECM 资源不足)"), ECM_MEMORY_INSUFFICIENT(11001, "ECM memory resources are insufficient(ECM 内存资源不足)"), diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/server/conf/EngineConnPluginConfiguration.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/server/conf/EngineConnPluginConfiguration.scala index de1add173a..9a16d9b9e6 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/server/conf/EngineConnPluginConfiguration.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/server/conf/EngineConnPluginConfiguration.scala @@ -35,4 +35,8 @@ object EngineConnPluginConfiguration { val ENABLED_BML_UPLOAD_FAILED_EXIT: CommonVars[Boolean] = CommonVars("wds.linkis.engineconn.bml.upload.failed.enable", true) + // for third party eg appconn/datax, if all update, can set to false then to remove + val EC_BML_VERSION_MAY_WITH_PREFIX_V: CommonVars[Boolean] = + CommonVars("linkis.engineconn.bml.version.may.with.prefix", true) + } diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/server/localize/AbstractEngineConnBmlResourceGenerator.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/server/localize/AbstractEngineConnBmlResourceGenerator.scala index fce3508c31..bba2f5d93d 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/server/localize/AbstractEngineConnBmlResourceGenerator.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/server/localize/AbstractEngineConnBmlResourceGenerator.scala @@ -17,6 +17,7 @@ package org.apache.linkis.engineplugin.server.localize +import org.apache.linkis.engineplugin.server.conf.EngineConnPluginConfiguration import org.apache.linkis.engineplugin.server.conf.EngineConnPluginConfiguration.ENGINE_CONN_HOME import org.apache.linkis.engineplugin.server.localize.EngineConnBmlResourceGenerator.NO_VERSION_MARK import org.apache.linkis.manager.engineplugin.common.exception.EngineConnPluginErrorException @@ -53,13 +54,38 @@ abstract class AbstractEngineConnBmlResourceGenerator extends EngineConnBmlResou val engineConnPackageHome = Paths.get(engineConnDistHome, version).toFile.getPath logger.info("getEngineConnDistHome, engineConnPackageHome path:" + engineConnPackageHome) val engineConnPackageHomeFile = new File(engineConnPackageHome) + + // 兼容老版本 if (!engineConnPackageHomeFile.exists()) { - throw new EngineConnPluginErrorException( - ENGINE_VERSION_NOT_FOUND.getErrorCode, - MessageFormat.format(ENGINE_VERSION_NOT_FOUND.getErrorDesc, version, engineConnType) - ) + if ( + !version.startsWith( + "v" + ) && EngineConnPluginConfiguration.EC_BML_VERSION_MAY_WITH_PREFIX_V.getValue + ) { + val versionOld = "v" + version + val engineConnPackageHomeOld = Paths.get(engineConnDistHome, versionOld).toFile.getPath + logger.info( + "try to getEngineConnDistHome with prefix v, engineConnPackageHome path:" + engineConnPackageHomeOld + ) + val engineConnPackageHomeFileOld = new File(engineConnPackageHomeOld) + if (!engineConnPackageHomeFileOld.exists()) { + throw new EngineConnPluginErrorException( + ENGINE_VERSION_NOT_FOUND.getErrorCode, + MessageFormat.format(ENGINE_VERSION_NOT_FOUND.getErrorDesc, version, engineConnType) + ) + } else { + engineConnPackageHomeOld + } + } else { + throw new EngineConnPluginErrorException( + ENGINE_VERSION_NOT_FOUND.getErrorCode, + MessageFormat.format(ENGINE_VERSION_NOT_FOUND.getErrorDesc, version, engineConnType) + ) + } + } else { + engineConnPackageHome } - engineConnPackageHome + } private def checkEngineConnDistHome(engineConnPackageHomePath: String): Unit = { @@ -71,7 +97,7 @@ abstract class AbstractEngineConnBmlResourceGenerator extends EngineConnBmlResou if (!engineConnPackageHome.exists()) { throw new EngineConnPluginErrorException( CANNOT_HOME_PATH_DIST.getErrorCode, - CANNOT_HOME_PATH_DIST.getErrorDesc + MessageFormat.format(CANNOT_HOME_PATH_DIST.getErrorDesc, engineConnPackageHome.getPath) ) } } diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/server/service/DefaultEngineConnResourceService.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/server/service/DefaultEngineConnResourceService.scala index 3ddfdc7bbe..1aba512f4c 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/server/service/DefaultEngineConnResourceService.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/server/service/DefaultEngineConnResourceService.scala @@ -34,15 +34,16 @@ import org.apache.linkis.manager.engineplugin.common.launch.process.{ LaunchConstants } import org.apache.linkis.manager.engineplugin.errorcode.EngineconnCoreErrorCodeSummary._ +import org.apache.linkis.publicservice.common.lock.entity.CommonLock +import org.apache.linkis.publicservice.common.lock.service.CommonLockService import org.apache.linkis.rpc.message.annotation.Receiver import org.apache.commons.lang3.StringUtils -import org.apache.commons.lang3.exception.ExceptionUtils import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Component -import javax.annotation.PostConstruct +import javax.annotation.{PostConstruct, PreDestroy} import java.text.MessageFormat import java.util.Date @@ -58,15 +59,44 @@ class DefaultEngineConnResourceService extends EngineConnResourceService with Lo @Autowired private var engineConnBmlResourceDao: EngineConnBmlResourceDao = _ + @Autowired + private val commonLockService: CommonLockService = null + private val bmlClient = BmlClientFactory.createBmlClient() private var isRefreshing: Boolean = false + private val _LOCK = "_MASTER_LOAD_ENGINE_CONN_LOCK" + val commonLock = new CommonLock + private var lock = false @PostConstruct - override def init(): Unit = - if (EngineConnPluginConfiguration.ENGINE_CONN_DIST_LOAD_ENABLE.getValue) { - logger.info("Start to refresh all engineconn plugins when inited.") + override def init(): Unit = { + commonLock.setLockObject(_LOCK) + commonLock.setCreateTime(new Date) + commonLock.setUpdateTime(new Date) + commonLock.setCreator(Utils.getJvmUser) + commonLock.setLocker(Utils.getLocalHostname) + commonLock.setUpdator(Utils.getJvmUser) + lock = commonLockService.reentrantLock(commonLock, -1) + if (lock) { + logger.info( + "The master LinkisManager node get lock by {}. And start to refresh all engineconn plugins when inited.", + _LOCK + "-" + commonLock.getLocker + ) refreshAll(false) } + } + + @PreDestroy + def destroy(): Unit = { + if (lock) { + logger.info( + "The master LinkisManager node hase released lock {}.", + commonLock.getLockObject + "-" + + commonLock.getLocker + ) + commonLockService.unlock(commonLock) + } + } private def uploadToBml(localizeResource: EngineConnLocalizeResource): BmlResource = { val response = bmlClient.uploadResource( @@ -201,8 +231,6 @@ class DefaultEngineConnResourceService extends EngineConnResourceService with Lo engineConnBmlResource.setFileName(localizeResource.fileName) engineConnBmlResource.setFileSize(localizeResource.fileSize) engineConnBmlResource.setLastModified(localizeResource.lastModified) - if (version.startsWith("v")) engineConnBmlResource.setVersion(version.substring(1)) - else engineConnBmlResource.setVersion(version) engineConnBmlResource.setVersion(version) engineConnBmlResourceDao.save(engineConnBmlResource) } else { @@ -241,9 +269,18 @@ class DefaultEngineConnResourceService extends EngineConnResourceService with Lo ): EngineConnResource = { val engineConnType = engineConnBMLResourceRequest.getEngineConnType val version = engineConnBMLResourceRequest.getVersion - val engineConnBmlResources = asScalaBufferConverter( + var engineConnBmlResources = asScalaBufferConverter( engineConnBmlResourceDao.getAllEngineConnBmlResource(engineConnType, version) ) + if ( + engineConnBmlResources.asScala.size == 0 && EngineConnPluginConfiguration.EC_BML_VERSION_MAY_WITH_PREFIX_V.getValue + ) { + logger.info("Try to get engine conn bml resource with prefex v") + engineConnBmlResources = asScalaBufferConverter( + engineConnBmlResourceDao.getAllEngineConnBmlResource(engineConnType, "v" + version) + ) + } + val confBmlResourceMap = engineConnBmlResources.asScala .find(_.getFileName == LaunchConstants.ENGINE_CONN_CONF_DIR_NAME + ".zip") .map(parseToBmlResource) diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/conf/AMConfiguration.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/conf/AMConfiguration.scala index c730edf0fd..edd3c001b5 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/conf/AMConfiguration.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/conf/AMConfiguration.scala @@ -23,6 +23,20 @@ import org.apache.linkis.manager.common.entity.enumeration.MaintainType object AMConfiguration { + val YARN_QUEUE_NAME_CONFIG_KEY = "wds.linkis.rm.yarnqueue" + + val ACROSS_CLUSTER_QUEUE_SUFFIX = "queueRuleSuffix" + + val ACROSS_CLUSTER_TASK = "acrossClusterTask" + + val ACROSS_CLUSTER_CPU_THRESHOLD = "CPUThreshold" + + val ACROSS_CLUSTER_MEMORY_THRESHOLD = "MemoryThreshold" + + val ACROSS_CLUSTER_CPU_PERCENTAGE_THRESHOLD = "CPUPercentageThreshold" + + val ACROSS_CLUSTER_MEMORY_PERCENTAGE_THRESHOLD = "MemoryPercentageThreshold" + val ECM_ADMIN_OPERATIONS = CommonVars("wds.linkis.governance.admin.operations", "") val ENGINE_START_MAX_TIME = diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/job/JobSubType.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/hook/AskEngineConnHook.scala similarity index 84% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/job/JobSubType.java rename to linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/hook/AskEngineConnHook.scala index c2c8539e4b..7240288698 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/job/JobSubType.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/hook/AskEngineConnHook.scala @@ -15,9 +15,11 @@ * limitations under the License. */ -package org.apache.linkis.cli.common.entity.job; +package org.apache.linkis.manager.am.hook -public interface JobSubType { +trait AskEngineConnHook { + + @throws(classOf[Exception]) + def doHook(ctx: AskEngineConnHookContext): Unit - String getName(); } diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/LinkisJob.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/hook/AskEngineConnHookContext.scala similarity index 72% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/LinkisJob.java rename to linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/hook/AskEngineConnHookContext.scala index a32521c20e..d5fcdb9686 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/LinkisJob.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/hook/AskEngineConnHookContext.scala @@ -15,12 +15,12 @@ * limitations under the License. */ -package org.apache.linkis.cli.application.interactor.job; +package org.apache.linkis.manager.am.hook -import org.apache.linkis.cli.application.interactor.job.data.LinkisJobData; -import org.apache.linkis.cli.core.interactor.job.AbstractJob; +import org.apache.linkis.manager.common.protocol.engine.EngineAskRequest +import org.apache.linkis.rpc.Sender -public abstract class LinkisJob extends AbstractJob { - @Override - public abstract LinkisJobData getJobData(); +class AskEngineConnHookContext(request: EngineAskRequest, sender: Sender) { + def getRequest(): EngineAskRequest = request + def getSender(): Sender = sender } diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/manager/DefaultEngineNodeManager.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/manager/DefaultEngineNodeManager.scala index 86cf986d29..68e7d4cbd9 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/manager/DefaultEngineNodeManager.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/manager/DefaultEngineNodeManager.scala @@ -25,7 +25,12 @@ import org.apache.linkis.manager.am.exception.{AMErrorCode, AMErrorException} import org.apache.linkis.manager.am.locker.EngineNodeLocker import org.apache.linkis.manager.common.constant.AMConstant import org.apache.linkis.manager.common.entity.enumeration.NodeStatus -import org.apache.linkis.manager.common.entity.node.{AMEngineNode, EngineNode, ScoreServiceInstance} +import org.apache.linkis.manager.common.entity.node.{ + AMEngineNode, + EngineNode, + RMNode, + ScoreServiceInstance +} import org.apache.linkis.manager.common.entity.persistence.PersistenceLabel import org.apache.linkis.manager.common.protocol.engine.{ EngineOperateRequest, @@ -42,6 +47,8 @@ import org.apache.linkis.manager.rm.service.ResourceManager import org.apache.linkis.manager.service.common.metrics.MetricsConverter import org.apache.linkis.manager.service.common.pointer.NodePointerBuilder +import org.apache.commons.lang3.StringUtils + import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Service @@ -49,6 +56,7 @@ import java.lang.reflect.UndeclaredThrowableException import java.util import scala.collection.JavaConverters._ +import scala.tools.scalap.scalax.util.StringUtil @Service class DefaultEngineNodeManager extends EngineNodeManager with Logging { @@ -105,7 +113,6 @@ class DefaultEngineNodeManager extends EngineNodeManager with Logging { val heartMsg = engine.getNodeHeartbeatMsg() engineNode.setNodeHealthyInfo(heartMsg.getHealthyInfo) engineNode.setNodeOverLoadInfo(heartMsg.getOverLoadInfo) - engineNode.setNodeResource(heartMsg.getNodeResource) engineNode.setNodeStatus(heartMsg.getStatus) engineNode } @@ -125,7 +132,9 @@ class DefaultEngineNodeManager extends EngineNodeManager with Logging { toState: NodeStatus ): Unit = {} - override def updateEngine(engineNode: EngineNode): Unit = {} + override def updateEngine(engineNode: EngineNode): Unit = { + nodeManagerPersistence.updateNodeInstance(engineNode) + } override def switchEngine(engineNode: EngineNode): EngineNode = { null @@ -133,7 +142,7 @@ class DefaultEngineNodeManager extends EngineNodeManager with Logging { override def reuseEngine(engineNode: EngineNode): EngineNode = { val node = getEngineNodeInfo(engineNode) - if (!NodeStatus.isAvailable(node.getNodeStatus)) { + if (node == null || !NodeStatus.isAvailable(node.getNodeStatus)) { return null } if (!NodeStatus.isLocked(node.getNodeStatus)) { @@ -164,7 +173,7 @@ class DefaultEngineNodeManager extends EngineNodeManager with Logging { retryHandler.addRetryException(classOf[UndeclaredThrowableException]) val node = retryHandler.retry[EngineNode](getEngineNodeInfo(engineNode), "getEngineNodeInfo") // val node = getEngineNodeInfo(engineNode) - if (!NodeStatus.isAvailable(node.getNodeStatus)) { + if (node == null || !NodeStatus.isAvailable(node.getNodeStatus)) { return null } if (!NodeStatus.isLocked(node.getNodeStatus)) { @@ -293,13 +302,16 @@ class DefaultEngineNodeManager extends EngineNodeManager with Logging { override def getEngineNodeInfo(serviceInstance: ServiceInstance): EngineNode = { val engineNode = getEngineNode(serviceInstance) if (engineNode != null) { + val nodeMetric = nodeMetricManagerPersistence.getNodeMetrics(engineNode) if (engineNode.getNodeStatus == null) { - val nodeMetric = nodeMetricManagerPersistence.getNodeMetrics(engineNode) engineNode.setNodeStatus( if (Option(nodeMetric).isDefined) NodeStatus.values()(nodeMetric.getStatus) else NodeStatus.Starting ) } + if (null != nodeMetric && StringUtils.isNotBlank(nodeMetric.getHeartBeatMsg)) { + engineNode.setEcMetrics(nodeMetric.getHeartBeatMsg) + } return engineNode } throw new AMErrorException( diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/pointer/DefaultEMNodPointer.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/pointer/DefaultEMNodPointer.scala index 5bd74edcde..8fc82f6c4b 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/pointer/DefaultEMNodPointer.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/pointer/DefaultEMNodPointer.scala @@ -17,6 +17,7 @@ package org.apache.linkis.manager.am.pointer +import org.apache.linkis.common.exception.LinkisRetryException import org.apache.linkis.common.utils.Utils import org.apache.linkis.manager.am.exception.AMErrorException import org.apache.linkis.manager.am.utils.AMUtils @@ -38,16 +39,17 @@ class DefaultEMNodPointer(val node: Node) extends AbstractNodePointer with EMNod override def createEngine(engineConnLaunchRequest: EngineConnLaunchRequest): EngineNode = { logger.info(s"Start to createEngine ask em ${getNode().getServiceInstance}") - getSender.ask(engineConnLaunchRequest) match { + val ec = getSender.ask(engineConnLaunchRequest) + ec match { case engineNode: EngineNode => logger.info( s"Succeed to createEngine ask em ${getNode().getServiceInstance}, engineNode $engineNode " ) engineNode case _ => - throw new AMErrorException( + throw new LinkisRetryException( AMConstant.ENGINE_ERROR_CODE, - s"Failed to createEngine ask em ${getNode().getServiceInstance}" + s"Failed to createEngine ask em ${getNode().getServiceInstance}, for return ${ec}" ) } } diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/pointer/DefaultEngineNodPointer.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/pointer/DefaultEngineNodPointer.scala index f1bcf8eb9a..eda42b6dbf 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/pointer/DefaultEngineNodPointer.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/pointer/DefaultEngineNodPointer.scala @@ -59,7 +59,8 @@ class DefaultEngineNodPointer(val node: Node) extends AbstractNodePointer with E override def executeOperation( engineOperateRequest: EngineOperateRequest ): EngineOperateResponse = { - getSender.ask(engineOperateRequest) match { + val rs = getSender.ask(engineOperateRequest) + rs match { case response: EngineOperateResponse => response case _ => throw new WarnException(-1, "Illegal response of operation.") } diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/em/DefaultEMEngineService.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/em/DefaultEMEngineService.scala index 918faf912f..f4af1c5550 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/em/DefaultEMEngineService.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/em/DefaultEMEngineService.scala @@ -19,21 +19,35 @@ package org.apache.linkis.manager.am.service.em import org.apache.linkis.common.utils.Logging import org.apache.linkis.engineplugin.server.service.EngineConnLaunchService +import org.apache.linkis.governance.common.utils.ECPathUtils import org.apache.linkis.manager.am.exception.AMErrorException import org.apache.linkis.manager.am.manager.{EMNodeManager, EngineNodeManager} +import org.apache.linkis.manager.am.service.ECResourceInfoService import org.apache.linkis.manager.am.service.EMEngineService import org.apache.linkis.manager.common.constant.AMConstant import org.apache.linkis.manager.common.entity.node._ +import org.apache.linkis.manager.common.entity.persistence.{ + ECResourceInfoRecord, + PersistenceResource +} import org.apache.linkis.manager.common.protocol.em._ import org.apache.linkis.manager.common.protocol.engine.EngineStopRequest import org.apache.linkis.manager.common.utils.ManagerUtils import org.apache.linkis.manager.engineplugin.common.launch.entity.EngineConnBuildRequest import org.apache.linkis.manager.label.entity.{EngineNodeLabel, Label} import org.apache.linkis.manager.label.entity.em.EMInstanceLabel +import org.apache.linkis.manager.label.entity.engine.{ + EngineInstanceLabel, + EngineTypeLabel, + UserCreatorLabel +} import org.apache.linkis.manager.label.service.NodeLabelService +import org.apache.linkis.manager.rm.domain.RMLabelContainer +import org.apache.linkis.manager.rm.service.LabelResourceService import org.apache.linkis.manager.service.common.label.LabelFilter import org.apache.commons.collections.MapUtils +import org.apache.commons.lang3.StringUtils import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Service @@ -57,9 +71,15 @@ class DefaultEMEngineService extends EMEngineService with Logging { @Autowired private var engineConnLaunchService: EngineConnLaunchService = _ + @Autowired + private var ecResourceInfoService: ECResourceInfoService = _ + @Autowired private var labelFilter: LabelFilter = _ + @Autowired + private var labelResourceService: LabelResourceService = _ + override def listEngines(getEMEnginesRequest: GetEMEnginesRequest): util.List[EngineNode] = { val emNode = new AMEMNode() emNode.setServiceInstance(getEMEnginesRequest.getEm) @@ -95,8 +115,54 @@ class DefaultEMEngineService extends EMEngineService with Logging { ) val engineStopRequest = new EngineStopRequest engineStopRequest.setServiceInstance(engineNode.getServiceInstance) + engineStopRequest.setIdentifierType(engineNode.getMark) + engineStopRequest.setIdentifier(engineNode.getIdentifier) + val ecResourceInfo: ECResourceInfoRecord = + if (StringUtils.isNotBlank(engineNode.getTicketId)) { + ecResourceInfoService.getECResourceInfoRecord(engineNode.getTicketId) + } else { + ecResourceInfoService.getECResourceInfoRecordByInstance( + engineNode.getServiceInstance.getInstance + ) + } + + if (ecResourceInfo != null) { + engineStopRequest.setEngineType(ecResourceInfo.getEngineType()) + engineStopRequest.setLogDirSuffix(ecResourceInfo.getLogDirSuffix) + } else { + if (engineNode.getLabels.isEmpty) { + // node labels is empty, engine already been stopped + logger.info( + s"DefaultEMEngineService stopEngine node labels is empty, engine: ${engineStopRequest.getServiceInstance} have already been stopped." + ) + return + } + + val rMLabelContainer: RMLabelContainer = + labelResourceService.enrichLabels(engineNode.getLabels) + + val persistenceResource: PersistenceResource = + labelResourceService.getPersistenceResource(rMLabelContainer.getEngineInstanceLabel) + if (persistenceResource == null) { + // persistenceResource is null, engine already been stopped + logger.info( + s"DefaultEMEngineService stopEngine persistenceResource is null, engine: ${engineStopRequest.getServiceInstance} have already been stopped." + ) + return + } + + engineStopRequest.setEngineType(rMLabelContainer.getEngineTypeLabel.getEngineType) + engineStopRequest.setLogDirSuffix( + ECPathUtils + .getECLogDirSuffix( + rMLabelContainer.getEngineTypeLabel, + rMLabelContainer.getUserCreatorLabel, + persistenceResource.getTicketId + ) + ) + } + emNodeManager.stopEngine(engineStopRequest, emNode) - // engineNodeManager.deleteEngineNode(engineNode) logger.info( s"EM ${emNode.getServiceInstance} finished to stop Engine ${engineNode.getServiceInstance}" ) @@ -116,7 +182,11 @@ class DefaultEMEngineService extends EMEngineService with Logging { val filterInstanceAndLabel = if (emInstanceLabelOption.isDefined) { val emInstanceLabel = emInstanceLabelOption.get.asInstanceOf[EMInstanceLabel] logger.info(s"use emInstanceLabel , will be route to ${emInstanceLabel.getServiceInstance}") - if (!instanceAndLabels.asScala.exists(_._1.equals(emInstanceLabel.getServiceInstance))) { + if ( + !instanceAndLabels.asScala.exists( + _._1.getServiceInstance.equals(emInstanceLabel.getServiceInstance) + ) + ) { throw new AMErrorException( AMConstant.EM_ERROR_CODE, s"You specified em ${emInstanceLabel.getServiceInstance}, but the corresponding EM does not exist in the Manager" diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/em/DefaultEMUnregisterService.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/em/DefaultEMUnregisterService.scala index b54815092f..cdcc670fdd 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/em/DefaultEMUnregisterService.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/em/DefaultEMUnregisterService.scala @@ -58,11 +58,6 @@ class DefaultEMUnregisterService extends EMUnregisterService with Logging { val emClearRequest = new EMInfoClearRequest emClearRequest.setEm(node) emClearRequest.setUser(stopEMRequest.getUser) - Utils.tryAndWarn(rmMessageService.dealWithStopEMRequest(stopEMRequest)) - // clear Label - val instanceLabelRemoveRequest = new NodeLabelRemoveRequest(node.getServiceInstance, false) - Utils.tryAndWarn(nodeLabelRemoveService.removeNodeLabel(instanceLabelRemoveRequest)) - // 此处需要先清理ECM再等待，避免ECM重启过快，导致ECM资源没清理干净 clearEMInstanceInfo(emClearRequest) logger.info(s" user ${stopEMRequest.getUser} finished to stop em ${stopEMRequest.getEm}") } diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineAskEngineService.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineAskEngineService.scala index 4616cf85a1..6ffff3ea2f 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineAskEngineService.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineAskEngineService.scala @@ -19,20 +19,25 @@ package org.apache.linkis.manager.am.service.engine import org.apache.linkis.common.exception.LinkisRetryException import org.apache.linkis.common.utils.{Logging, Utils} -import org.apache.linkis.governance.common.utils.JobUtils +import org.apache.linkis.governance.common.utils.{JobUtils, LoggerUtils} import org.apache.linkis.manager.am.conf.AMConfiguration +import org.apache.linkis.manager.am.hook.{AskEngineConnHook, AskEngineConnHookContext} +import org.apache.linkis.manager.am.service.engine.EngineAskEngineService.getAsyncId import org.apache.linkis.manager.common.constant.AMConstant import org.apache.linkis.manager.common.protocol.engine._ import org.apache.linkis.manager.label.constant.LabelKeyConstant import org.apache.linkis.rpc.Sender import org.apache.linkis.rpc.message.annotation.Receiver +import org.apache.commons.lang3.StringUtils import org.apache.commons.lang3.exception.ExceptionUtils -import org.springframework.beans.factory.annotation.Autowired +import org.springframework.beans.factory.annotation.{Autowired, Qualifier} import org.springframework.stereotype.Service import java.net.SocketTimeoutException +import java.util +import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.atomic.AtomicInteger import scala.concurrent._ @@ -55,9 +60,10 @@ class DefaultEngineAskEngineService @Autowired private var engineSwitchService: EngineSwitchService = _ - private val idCreator = new AtomicInteger() - - private val idPrefix = Sender.getThisServiceInstance.getInstance + @Autowired(required = false) + @Qualifier + /* The implementation class of hook must be annotated with @Qualifier to take effect(hook的实现类必须加上@Qualifier注解才能生效) */ + var hooksArray: Array[AskEngineConnHook] = _ private implicit val executor: ExecutionContextExecutorService = Utils.newCachedExecutionContext( @@ -67,7 +73,38 @@ class DefaultEngineAskEngineService @Receiver override def askEngine(engineAskRequest: EngineAskRequest, sender: Sender): Any = { + + if (hooksArray != null && hooksArray.size > 0) { + val ctx = new AskEngineConnHookContext(engineAskRequest, sender) + + /** Throwing exceptions in hook will block(hook中抛异常会阻断) */ + hooksArray.foreach(h => + Utils.tryCatch(h.doHook(ctx)) { t => + { + val engineAskAsyncId = getAsyncId + val retryFlag = t match { + case _: LinkisRetryException => true + case _: RetryableException => true + case _ => + ExceptionUtils.getRootCause(t) match { + case _: SocketTimeoutException => true + case _: TimeoutException => true + case _ => + false + } + } + return EngineCreateError( + engineAskAsyncId, + ExceptionUtils.getRootCauseMessage(t), + retryFlag + ) + } + } + ) + } + val taskId = JobUtils.getJobIdFromStringMap(engineAskRequest.getProperties) + LoggerUtils.setJobIdMDC(taskId) logger.info(s"received task: $taskId, engineAskRequest $engineAskRequest") if (!engineAskRequest.getLabels.containsKey(LabelKeyConstant.EXECUTE_ONCE_KEY)) { val engineReuseRequest = new EngineReuseRequest() @@ -91,16 +128,18 @@ class DefaultEngineAskEngineService logger.info( s"Finished to ask engine for task: $taskId user ${engineAskRequest.getUser} by reuse node $reuseNode" ) + LoggerUtils.removeJobIdMDC() return reuseNode } } val engineAskAsyncId = getAsyncId val createNodeThread = Future { + LoggerUtils.setJobIdMDC(taskId) logger.info( s"Task: $taskId start to async($engineAskAsyncId) createEngine, ${engineAskRequest.getCreateService}" ) - // 如果原来的labels含engineInstance ，先去掉 + // If the original labels contain engineInstance, remove it first (如果原来的labels含engineInstance ，先去掉) engineAskRequest.getLabels.remove("engineInstance") val engineCreateRequest = new EngineCreateRequest engineCreateRequest.setLabels(engineAskRequest.getLabels) @@ -108,30 +147,44 @@ class DefaultEngineAskEngineService engineCreateRequest.setUser(engineAskRequest.getUser) engineCreateRequest.setProperties(engineAskRequest.getProperties) engineCreateRequest.setCreateService(engineAskRequest.getCreateService) - val createNode = engineCreateService.createEngine(engineCreateRequest, sender) - val timeout = - if (engineCreateRequest.getTimeout <= 0) { - AMConfiguration.ENGINE_START_MAX_TIME.getValue.toLong - } else engineCreateRequest.getTimeout - // useEngine 需要加上超时 - val createEngineNode = getEngineNodeManager.useEngine(createNode, timeout) - if (null == createEngineNode) { - throw new LinkisRetryException( - AMConstant.EM_ERROR_CODE, - s"create engine${createNode.getServiceInstance} success, but to use engine failed" + Utils.tryFinally { + val createNode = engineCreateService.createEngine(engineCreateRequest, sender) + val timeout = + if (engineCreateRequest.getTimeout <= 0) { + AMConfiguration.ENGINE_START_MAX_TIME.getValue.toLong + } else engineCreateRequest.getTimeout + // UseEngine requires a timeout (useEngine 需要加上超时) + val createEngineNode = getEngineNodeManager.useEngine(createNode, timeout) + if (null == createEngineNode) { + throw new LinkisRetryException( + AMConstant.EM_ERROR_CODE, + s"create engine${createNode.getServiceInstance} success, but to use engine failed" + ) + } + logger.info( + s"Task: $taskId finished to ask engine for user ${engineAskRequest.getUser} by create node $createEngineNode" ) + createEngineNode + } { + LoggerUtils.removeJobIdMDC() } - logger.info( - s"Task: $taskId finished to ask engine for user ${engineAskRequest.getUser} by create node $createEngineNode" - ) - createEngineNode } createNodeThread.onComplete { case Success(engineNode) => - logger.info(s"Task: $taskId Success to async($engineAskAsyncId) createEngine $engineNode") - sender.send(EngineCreateSuccess(engineAskAsyncId, engineNode)) + LoggerUtils.setJobIdMDC(taskId) + Utils.tryFinally { + logger.info(s"Task: $taskId Success to async($engineAskAsyncId) createEngine $engineNode") + if (null != sender) { + sender.send(EngineCreateSuccess(engineAskAsyncId, engineNode)) + } else { + logger.info("Will not send async useing null sender.") + } + } { + LoggerUtils.removeJobIdMDC() + } case Failure(exception) => + LoggerUtils.setJobIdMDC(taskId) val retryFlag = exception match { case retryException: LinkisRetryException => true case retryableException: RetryableException => true @@ -151,20 +204,20 @@ class DefaultEngineAskEngineService logger.info(s"msg: ${msg} canRetry Exception: ${exception.getClass.getName}") } - sender.send( - EngineCreateError( - engineAskAsyncId, - ExceptionUtils.getRootCauseMessage(exception), - retryFlag + Utils.tryFinally { + sender.send( + EngineCreateError( + engineAskAsyncId, + ExceptionUtils.getRootCauseMessage(exception), + retryFlag + ) ) - ) + } { + LoggerUtils.removeJobIdMDC() + } } - + LoggerUtils.removeJobIdMDC() EngineAskAsyncResponse(engineAskAsyncId, Sender.getThisServiceInstance) } - private def getAsyncId: String = { - idPrefix + "_" + idCreator.getAndIncrement() - } - } diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineConnStatusCallbackService.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineConnStatusCallbackService.scala deleted file mode 100644 index ccfcb7aa28..0000000000 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineConnStatusCallbackService.scala +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.manager.am.service.engine - -import org.apache.linkis.common.utils.Logging -import org.apache.linkis.manager.am.conf.AMConfiguration -import org.apache.linkis.manager.common.constant.AMConstant -import org.apache.linkis.manager.common.entity.metrics.AMNodeMetrics -import org.apache.linkis.manager.common.protocol.engine.EngineConnStatusCallbackToAM -import org.apache.linkis.manager.persistence.NodeMetricManagerPersistence -import org.apache.linkis.manager.service.common.metrics.MetricsConverter -import org.apache.linkis.rpc.message.annotation.Receiver -import org.apache.linkis.server.BDPJettyServerHelper - -import org.apache.commons.lang3.StringUtils - -import org.springframework.beans.factory.annotation.Autowired -import org.springframework.stereotype.Service - -import java.util -import java.util.Locale - -@Service -class DefaultEngineConnStatusCallbackService extends EngineConnStatusCallbackService with Logging { - - @Autowired - private var nodeMetricManagerPersistence: NodeMetricManagerPersistence = _ - - @Autowired - private var metricsConverter: MetricsConverter = _ - - private val canRetryLogs = AMConfiguration.AM_CAN_RETRY_LOGS.getValue.split(";") - - // The heartBeatMsg field is of type text, mysql text max byte num is 65535 - private val initErrorMsgMaxByteNum = 60000 - - @Receiver - override def dealEngineConnStatusCallback( - engineConnStatusCallbackToAM: EngineConnStatusCallbackToAM - ): Unit = { - - if (null == engineConnStatusCallbackToAM.serviceInstance) { - logger.warn(s"call back service instance is null") - } - logger.info(s"Start to deal engineConnStatusCallbackToAM $engineConnStatusCallbackToAM") - val nodeMetrics = new AMNodeMetrics - val heartBeatMsg: java.util.Map[String, Any] = new util.HashMap[String, Any]() - - var initErrorMsg = engineConnStatusCallbackToAM.initErrorMsg - if ( - StringUtils.isNotBlank(initErrorMsg) && initErrorMsg - .getBytes("utf-8") - .length >= initErrorMsgMaxByteNum - ) { - initErrorMsg = initErrorMsg.substring(0, initErrorMsgMaxByteNum) - } - heartBeatMsg.put(AMConstant.START_REASON, initErrorMsg) - - if (engineConnStatusCallbackToAM.canRetry) { - heartBeatMsg.put(AMConstant.EC_CAN_RETRY, engineConnStatusCallbackToAM.canRetry) - } else if (matchRetryLog(engineConnStatusCallbackToAM.initErrorMsg)) { - logger.info(s"match canRetry log ${engineConnStatusCallbackToAM.serviceInstance}") - heartBeatMsg.put(AMConstant.EC_CAN_RETRY, engineConnStatusCallbackToAM.canRetry) - } - - nodeMetrics.setHeartBeatMsg(BDPJettyServerHelper.jacksonJson.writeValueAsString(heartBeatMsg)) - nodeMetrics.setServiceInstance(engineConnStatusCallbackToAM.serviceInstance) - nodeMetrics.setStatus(metricsConverter.convertStatus(engineConnStatusCallbackToAM.status)) - - nodeMetricManagerPersistence.addOrupdateNodeMetrics(nodeMetrics) - logger.info(s"Finished to deal engineConnStatusCallbackToAM $engineConnStatusCallbackToAM") - - } - - private def matchRetryLog(errorMsg: String): Boolean = { - var flag = false - if (StringUtils.isNotBlank(errorMsg)) { - val errorMsgLowCase = errorMsg.toLowerCase(Locale.getDefault) - canRetryLogs.foreach(canRetry => - if (errorMsgLowCase.contains(canRetry)) { - logger.info(s"match engineConn log fatal logs,is $canRetry") - flag = true - } - ) - } - flag - } - -} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineCreateService.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineCreateService.scala index d6ceccef96..78a8adefd2 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineCreateService.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineCreateService.scala @@ -18,6 +18,7 @@ package org.apache.linkis.manager.am.service.engine import org.apache.linkis.common.ServiceInstance +import org.apache.linkis.common.conf.CommonVars import org.apache.linkis.common.exception.LinkisRetryException import org.apache.linkis.common.utils.{ByteTimeUtils, Logging, Utils} import org.apache.linkis.engineplugin.server.service.EngineConnResourceFactoryService @@ -291,6 +292,17 @@ class DefaultEngineCreateService } }) } + + val queueRuleSuffix = props.get(AMConfiguration.ACROSS_CLUSTER_QUEUE_SUFFIX) + if (StringUtils.isNotBlank(queueRuleSuffix)) { + val queueName = props.getOrDefault(AMConfiguration.YARN_QUEUE_NAME_CONFIG_KEY, "default") + val newQueueName = queueName + "_" + queueRuleSuffix + props.put(AMConfiguration.YARN_QUEUE_NAME_CONFIG_KEY, newQueueName) + logger.info( + s"Switch queues according to queueRule with queue name : $queueName to $newQueueName" + ) + } + val timeoutEngineResourceRequest = TimeoutEngineResourceRequest( timeout, engineCreateRequest.getUser, @@ -303,13 +315,14 @@ class DefaultEngineCreateService resourceManager.requestResource( LabelUtils.distinctLabel(labelList, emNode.getLabels), resource, + engineCreateRequest, timeout ) match { case AvailableResource(ticketId) => (ticketId, resource) case NotEnoughResource(reason) => - logger.warn(s"not engough resource: $reason") - throw new LinkisRetryException(AMConstant.EM_ERROR_CODE, s"not engough resource: : $reason") + logger.warn(s"not enough resource: $reason") + throw new LinkisRetryException(AMConstant.EM_ERROR_CODE, s"not enough resource: : $reason") } } @@ -327,7 +340,8 @@ class DefaultEngineCreateService if (null == engineNodeInfo) return false if (NodeStatus.isCompleted(engineNodeInfo.getNodeStatus)) { val metrics = nodeMetricManagerPersistence.getNodeMetrics(engineNodeInfo) - val (reason, canRetry) = getStartErrorInfo(metrics.getHeartBeatMsg) + val msg = if (metrics != null) metrics.getHeartBeatMsg else null + val (reason, canRetry) = getStartErrorInfo(msg) if (canRetry.isDefined) { throw new LinkisRetryException( AMConstant.ENGINE_ERROR_CODE, diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineStopService.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineStopService.scala index 801114c4bf..769697a13c 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineStopService.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineStopService.scala @@ -94,13 +94,17 @@ class DefaultEngineStopService extends AbstractEngineService with EngineStopServ logger.info(s" engineConn does not exist in db: $engineStopRequest ") return } + + val labels = nodeLabelService.getNodeLabels(engineStopRequest.getServiceInstance) + node.setLabels(labels) + // 1. request em to kill ec logger.info(s"Start to kill engine invoke enginePointer ${node.getServiceInstance}") Utils.tryAndErrorMsg { getEMService().stopEngine(node, node.getEMNode) logger.info(s"Finished to kill engine invoke enginePointer ${node.getServiceInstance}") }(s"Failed to stop engine ${node.getServiceInstance}") - node.setLabels(nodeLabelService.getNodeLabels(engineStopRequest.getServiceInstance)) + if (null == node.getNodeStatus) { node.setNodeStatus(NodeStatus.ShuttingDown) } @@ -191,16 +195,7 @@ class DefaultEngineStopService extends AbstractEngineService with EngineStopServ override def engineConnInfoClear(ecNode: EngineNode): Unit = { logger.info(s"Start to clear ec info $ecNode") // 1. to clear engine resource - Utils.tryCatch { - resourceManager.resourceReleased(ecNode) - } { - case exception: RMErrorException => - if (exception.getErrCode != RMErrorCode.LABEL_RESOURCE_NOT_FOUND.getErrorCode) { - throw exception - } - case exception: Exception => throw exception - } - + Utils.tryAndWarn { resourceManager.resourceReleased(ecNode) } // 2. to clear Label val instanceLabelRemoveRequest = new NodeLabelRemoveRequest(ecNode.getServiceInstance, true) nodeLabelRemoveService.removeNodeLabel(instanceLabelRemoveRequest) diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/EngineAskEngineService.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/EngineAskEngineService.scala index 43b1f677b3..2c21bf851f 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/EngineAskEngineService.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/EngineAskEngineService.scala @@ -17,11 +17,24 @@ package org.apache.linkis.manager.am.service.engine -import org.apache.linkis.manager.common.protocol.engine.EngineAskRequest +import org.apache.linkis.manager.common.protocol.engine.{EngineAskRequest, EngineAsyncResponse} import org.apache.linkis.rpc.Sender +import java.util.concurrent.atomic.AtomicInteger + trait EngineAskEngineService { def askEngine(engineAskRequest: EngineAskRequest, sender: Sender): Any } + +object EngineAskEngineService { + private val idCreator = new AtomicInteger() + + private val idPrefix = Sender.getThisServiceInstance.getInstance + + def getAsyncId: String = { + idPrefix + "_" + idCreator.getAndIncrement() + } + +} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/EngineConnStatusCallbackService.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/EngineConnStatusCallbackService.scala deleted file mode 100644 index b4e498155c..0000000000 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/EngineConnStatusCallbackService.scala +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.manager.am.service.engine - -import org.apache.linkis.manager.common.protocol.engine.EngineConnStatusCallbackToAM - -trait EngineConnStatusCallbackService { - - def dealEngineConnStatusCallback(engineConnStatusCallbackToAM: EngineConnStatusCallbackToAM): Unit - -} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/EngineCreateService.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/EngineCreateService.scala index 8e90e222c9..c1efd0f964 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/EngineCreateService.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/EngineCreateService.scala @@ -17,7 +17,6 @@ package org.apache.linkis.manager.am.service.engine -import org.apache.linkis.common.ServiceInstance import org.apache.linkis.common.exception.LinkisRetryException import org.apache.linkis.manager.common.entity.node.EngineNode import org.apache.linkis.manager.common.protocol.engine.EngineCreateRequest diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/EngineStopService.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/EngineStopService.scala index 228f3e0801..beca547906 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/EngineStopService.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/EngineStopService.scala @@ -17,7 +17,6 @@ package org.apache.linkis.manager.am.service.engine -import org.apache.linkis.manager.am.vo.ResourceVo import org.apache.linkis.manager.common.entity.node.EngineNode import org.apache.linkis.manager.common.protocol.engine.{ EngineConnReleaseRequest, @@ -26,10 +25,6 @@ import org.apache.linkis.manager.common.protocol.engine.{ } import org.apache.linkis.rpc.Sender -import java.util - -import scala.collection.mutable - trait EngineStopService { def stopEngine(engineStopRequest: EngineStopRequest, sender: Sender): Unit diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/heartbeat/AMHeartbeatService.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/heartbeat/AMHeartbeatService.scala index 25db0b8303..c951e01052 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/heartbeat/AMHeartbeatService.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/heartbeat/AMHeartbeatService.scala @@ -18,7 +18,6 @@ package org.apache.linkis.manager.am.service.heartbeat import org.apache.linkis.common.utils.{Logging, Utils} -import org.apache.linkis.manager.am.conf.ManagerMonitorConf import org.apache.linkis.manager.am.service.HeartbeatService import org.apache.linkis.manager.common.conf.RMConfiguration import org.apache.linkis.manager.common.entity.metrics.AMNodeMetrics @@ -26,13 +25,16 @@ import org.apache.linkis.manager.common.monitor.ManagerMonitor import org.apache.linkis.manager.common.protocol.node.NodeHeartbeatMsg import org.apache.linkis.manager.persistence.{NodeManagerPersistence, NodeMetricManagerPersistence} import org.apache.linkis.manager.service.common.metrics.MetricsConverter +import org.apache.linkis.publicservice.common.lock.entity.CommonLock +import org.apache.linkis.publicservice.common.lock.service.CommonLockService import org.apache.linkis.rpc.message.annotation.Receiver import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Service -import javax.annotation.PostConstruct +import javax.annotation.{PostConstruct, PreDestroy} +import java.util.Date import java.util.concurrent.TimeUnit @Service @@ -50,17 +52,43 @@ class AMHeartbeatService extends HeartbeatService with Logging { @Autowired(required = false) private var managerMonitor: ManagerMonitor = _ + @Autowired private val commonLockService: CommonLockService = null + + private val _LOCK = "_MASTER_AM_HEARTBEAT_MONITOR_LOCK" + val commonLock = new CommonLock + private var lock = false + @PostConstruct def init(): Unit = { - if (null != managerMonitor && ManagerMonitorConf.MONITOR_SWITCH_ON.getValue) { - logger.info("start init AMHeartbeatService monitor") + commonLock.setLockObject(_LOCK) + commonLock.setCreateTime(new Date) + commonLock.setUpdateTime(new Date) + commonLock.setCreator(Utils.getJvmUser) + commonLock.setLocker(Utils.getLocalHostname) + commonLock.setUpdator(Utils.getJvmUser) + lock = commonLockService.reentrantLock(commonLock, -1) + if (null != managerMonitor && lock) { + logger.info( + "The master am get lock by {}. And start to init AMHeartbeatService monitor.", + _LOCK + "-" + commonLock.getLocker + ) Utils.defaultScheduler.scheduleAtFixedRate( managerMonitor, 1000, RMConfiguration.RM_ENGINE_SCAN_INTERVAL.getValue.toLong, TimeUnit.MILLISECONDS ) + } + } + @PreDestroy + def destroy(): Unit = { + if (lock) { + commonLockService.unlock(commonLock) + logger.info( + "The master am has released lock {}.", + commonLock.getLockObject + "-" + commonLock.getLocker + ); } } @@ -78,6 +106,7 @@ class AMHeartbeatService extends HeartbeatService with Logging { nodeMetrics.setStatus(0) } nodeMetricManagerPersistence.addOrupdateNodeMetrics(nodeMetrics) + logger.info(s"Finished to deal nodeHeartbeatMsg $nodeHeartbeatMsg") } } diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/label/conf/LabelManagerConf.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/label/conf/LabelManagerConf.scala index 1e34f84d2d..12f1a26782 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/label/conf/LabelManagerConf.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/label/conf/LabelManagerConf.scala @@ -22,6 +22,6 @@ import org.apache.linkis.common.conf.CommonVars object LabelManagerConf { val LONG_LIVED_LABEL = - CommonVars("wds.linkis.label.node.long.lived.label.keys", "tenant").getValue + CommonVars("wds.linkis.label.node.long.lived.label.keys", "tenant|yarnCluster").getValue } diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/label/service/impl/DefaultNodeLabelService.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/label/service/impl/DefaultNodeLabelService.scala index 0ccd7fd06f..a8dbe44352 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/label/service/impl/DefaultNodeLabelService.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/label/service/impl/DefaultNodeLabelService.scala @@ -26,7 +26,7 @@ import org.apache.linkis.manager.label.LabelManagerUtils import org.apache.linkis.manager.label.builder.factory.LabelBuilderFactoryContext import org.apache.linkis.manager.label.conf.LabelManagerConf import org.apache.linkis.manager.label.entity.{Feature, Label} -import org.apache.linkis.manager.label.score.NodeLabelScorer +import org.apache.linkis.manager.label.score.{LabelScoreServiceInstance, NodeLabelScorer} import org.apache.linkis.manager.label.service.NodeLabelService import org.apache.linkis.manager.label.utils.LabelUtils import org.apache.linkis.manager.persistence.LabelManagerPersistence @@ -312,6 +312,8 @@ class DefaultNodeLabelService extends NodeLabelService with Logging { instanceLabels.keys } + val matchInstanceAndLabels = new util.HashMap[ScoreServiceInstance, util.List[Label[_]]]() + // Get the out-degree relations ( Node -> Label ) val outNodeDegree = labelManagerPersistence.getLabelRelationsByServiceInstance(instances.toList.asJava) @@ -322,12 +324,15 @@ class DefaultNodeLabelService extends NodeLabelService with Logging { else { necessaryLabels.asScala.map(_.getLabelKey).toSet } - // Rebuild in-degree relations - inNodeDegree.clear() - val removeNodes = new ArrayBuffer[ServiceInstance]() - outNodeDegree.asScala.foreach { case (node, iLabels) => - // The core tag must be exactly the same - if (null != necessaryLabels) { + if (null == necessaryLabels || necessaryLabels.isEmpty) { + outNodeDegree.asScala.foreach { case (node, iLabels) => + matchInstanceAndLabels.put( + new LabelScoreServiceInstance(node), + iLabels.asInstanceOf[util.List[Label[_]]] + ) + } + } else { + outNodeDegree.asScala.foreach { case (node, iLabels) => val coreLabelKeys = iLabels.asScala .map(ManagerUtils.persistenceLabelToRealLabel) .filter(_.getFeature == Feature.CORE) @@ -338,33 +343,21 @@ class DefaultNodeLabelService extends NodeLabelService with Logging { coreLabelKeys.asJava ) && coreLabelKeys.size == necessaryLabelKeys.size ) { - iLabels.asScala.foreach(label => { - if (!inNodeDegree.asScala.contains(label)) { - val inNodes = new util.ArrayList[ServiceInstance]() - inNodeDegree.put(label, inNodes) - } - val inNodes = inNodeDegree.get(label) - inNodes.add(node) - }) - } else { - removeNodes += node + matchInstanceAndLabels.put( + new LabelScoreServiceInstance(node), + iLabels.asInstanceOf[util.List[Label[_]]] + ) } } } - - // Remove nodes with mismatched labels - if (removeNodes.nonEmpty && removeNodes.size == outNodeDegree.size()) { - logger.info( - s"The entered labels${necessaryLabels} do not match the labels of the node itself" - ) - } - - removeNodes.foreach(outNodeDegree.remove(_)) - return nodeLabelScorer - .calculate(inNodeDegree, outNodeDegree, labels) - .asInstanceOf[util.Map[ScoreServiceInstance, util.List[Label[_]]]] } - new util.HashMap[ScoreServiceInstance, util.List[Label[_]]]() + // Remove nodes with mismatched labels + if (matchInstanceAndLabels.isEmpty) { + logger.info( + s"The entered labels${necessaryLabels} do not match the labels of the node itself" + ) + } + matchInstanceAndLabels } private def tryToAddLabel(persistenceLabel: PersistenceLabel): Int = { diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/external/yarn/YarnResourceRequester.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/external/yarn/YarnResourceRequester.scala index d7b3349cc7..a6ed7c7512 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/external/yarn/YarnResourceRequester.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/external/yarn/YarnResourceRequester.scala @@ -57,12 +57,11 @@ class YarnResourceRequester extends ExternalResourceRequester with Logging { private val HASTATE_ACTIVE = "active" - private var provider: ExternalResourceProvider = _ private val rmAddressMap: util.Map[String, String] = new ConcurrentHashMap[String, String]() - private def getAuthorizationStr = { - val user = this.provider.getConfigMap.getOrDefault("user", "").asInstanceOf[String] - val pwd = this.provider.getConfigMap.getOrDefault("pwd", "").asInstanceOf[String] + private def getAuthorizationStr(provider: ExternalResourceProvider) = { + val user = provider.getConfigMap.getOrDefault("user", "").asInstanceOf[String] + val pwd = provider.getConfigMap.getOrDefault("pwd", "").asInstanceOf[String] val authKey = user + ":" + pwd Base64.getMimeEncoder.encodeToString(authKey.getBytes) } @@ -71,9 +70,7 @@ class YarnResourceRequester extends ExternalResourceRequester with Logging { identifier: ExternalResourceIdentifier, provider: ExternalResourceProvider ): NodeResource = { - val rmWebHaAddress = provider.getConfigMap.get("rmWebAddress").asInstanceOf[String] - this.provider = provider - val rmWebAddress = getAndUpdateActiveRmWebAddress(rmWebHaAddress) + val rmWebAddress = getAndUpdateActiveRmWebAddress(provider) logger.info(s"rmWebAddress: $rmWebAddress") val queueName = identifier.asInstanceOf[YarnResourceIdentifier].getQueueName @@ -87,7 +84,7 @@ class YarnResourceRequester extends ExternalResourceRequester with Logging { ) def maxEffectiveHandle(queueValue: Option[JValue]): Option[YarnResource] = { - val metrics = getResponseByUrl("metrics", rmWebAddress) + val metrics = getResponseByUrl("metrics", rmWebAddress, provider) val totalResouceInfoResponse = ( (metrics \ "clusterMetrics" \ "totalMB").asInstanceOf[JInt].values.toLong, (metrics \ "clusterMetrics" \ "totalVirtualCores").asInstanceOf[JInt].values.toLong @@ -180,34 +177,47 @@ class YarnResourceRequester extends ExternalResourceRequester with Logging { def getChildQueuesOfCapacity(resp: JValue) = resp \ "queues" \ "queue" def getResources() = { - val resp = getResponseByUrl("scheduler", rmWebAddress) + val resp = getResponseByUrl("scheduler", rmWebAddress, provider) val schedulerType = (resp \ "scheduler" \ "schedulerInfo" \ "type").asInstanceOf[JString].values if ("capacityScheduler".equals(schedulerType)) { realQueueName = queueName val childQueues = getChildQueuesOfCapacity(resp \ "scheduler" \ "schedulerInfo") val queue = getQueueOfCapacity(childQueues) - if (queue.isEmpty) { - logger.debug(s"cannot find any information about queue $queueName, response: " + resp) - throw new RMWarnException( - YARN_NOT_EXISTS_QUEUE.getErrorCode, - MessageFormat.format(YARN_NOT_EXISTS_QUEUE.getErrorDesc, queueName) - ) + val queueOption = Option(queue) match { + case Some(queue) => queue + case None => + logger.debug(s"cannot find any information about queue $queueName, response: " + resp) + throw new RMWarnException( + YARN_NOT_EXISTS_QUEUE.getErrorCode, + MessageFormat.format(YARN_NOT_EXISTS_QUEUE.getErrorDesc, queueName) + ) } - (maxEffectiveHandle(queue).get, getYarnResource(queue.map(_ \ "resourcesUsed")).get) + val queueInfo = queueOption.get.asInstanceOf[JObject] + ( + maxEffectiveHandle(queue).get, + getYarnResource(queue.map(_ \ "resourcesUsed")).get, + (queueInfo \ "maxApps").asInstanceOf[JInt].values.toInt, + (queueInfo \ "numPendingApps").asInstanceOf[JInt].values.toInt, + (queueInfo \ "numActiveApps").asInstanceOf[JInt].values.toInt + ) } else if ("fairScheduler".equals(schedulerType)) { val childQueues = getChildQueues(resp \ "scheduler" \ "schedulerInfo" \ "rootQueue") val queue = getQueue(childQueues) - if (queue.isEmpty) { + if (queue.isEmpty || queue.get == null) { logger.debug(s"cannot find any information about queue $queueName, response: " + resp) throw new RMWarnException( YARN_NOT_EXISTS_QUEUE.getErrorCode, MessageFormat.format(YARN_NOT_EXISTS_QUEUE.getErrorDesc, queueName) ) } + val queueInfo = queue.get.asInstanceOf[JObject] ( getYarnResource(queue.map(_ \ "maxResources")).get, - getYarnResource(queue.map(_ \ "usedResources")).get + getYarnResource(queue.map(_ \ "usedResources")).get, + (queueInfo \ "maxApps").asInstanceOf[JInt].values.toInt, + (queueInfo \ "numPendingApps").asInstanceOf[JInt].values.toInt, + (queueInfo \ "numActiveApps").asInstanceOf[JInt].values.toInt ) } else { logger.debug( @@ -225,6 +235,9 @@ class YarnResourceRequester extends ExternalResourceRequester with Logging { val nodeResource = new CommonNodeResource nodeResource.setMaxResource(yarnResource._1) nodeResource.setUsedResource(yarnResource._2) + nodeResource.setMaxApps(yarnResource._3) + nodeResource.setNumPendingApps(yarnResource._4) + nodeResource.setNumActiveApps(yarnResource._5) nodeResource }(t => { throw new RMErrorException( @@ -239,9 +252,8 @@ class YarnResourceRequester extends ExternalResourceRequester with Logging { identifier: ExternalResourceIdentifier, provider: ExternalResourceProvider ): java.util.List[ExternalAppInfo] = { - val rmWebHaAddress = provider.getConfigMap.get("rmWebAddress").asInstanceOf[String] - val rmWebAddress = getAndUpdateActiveRmWebAddress(rmWebHaAddress) + val rmWebAddress = getAndUpdateActiveRmWebAddress(provider) val queueName = identifier.asInstanceOf[YarnResourceIdentifier].getQueueName @@ -257,7 +269,7 @@ class YarnResourceRequester extends ExternalResourceRequester with Logging { val realQueueName = "root." + queueName def getAppInfos(): Array[ExternalAppInfo] = { - val resp = getResponseByUrl("apps", rmWebAddress) + val resp = getResponseByUrl("apps", rmWebAddress, provider) resp \ "apps" \ "app" match { case JArray(apps) => val appInfoBuffer = new ArrayBuffer[YarnAppInfo]() @@ -291,25 +303,29 @@ class YarnResourceRequester extends ExternalResourceRequester with Logging { override def getResourceType: ResourceType = ResourceType.Yarn - private def getResponseByUrl(url: String, rmWebAddress: String) = { + private def getResponseByUrl( + url: String, + rmWebAddress: String, + provider: ExternalResourceProvider + ) = { val httpGet = new HttpGet(rmWebAddress + "/ws/v1/cluster/" + url) httpGet.addHeader("Accept", "application/json") - val authorEnable: Any = this.provider.getConfigMap.get("authorEnable"); + val authorEnable: Any = provider.getConfigMap.get("authorEnable"); var httpResponse: HttpResponse = null authorEnable match { case flag: Boolean => if (flag) { - httpGet.addHeader(HttpHeaders.AUTHORIZATION, "Basic " + getAuthorizationStr) + httpGet.addHeader(HttpHeaders.AUTHORIZATION, "Basic " + getAuthorizationStr(provider)) } case _ => } - val kerberosEnable: Any = this.provider.getConfigMap.get("kerberosEnable"); + val kerberosEnable: Any = provider.getConfigMap.get("kerberosEnable"); kerberosEnable match { case flag: Boolean => if (flag) { - val principalName = this.provider.getConfigMap.get("principalName").asInstanceOf[String] - val keytabPath = this.provider.getConfigMap.get("keytabPath").asInstanceOf[String] - val krb5Path = this.provider.getConfigMap.get("krb5Path").asInstanceOf[String] + val principalName = provider.getConfigMap.get("principalName").asInstanceOf[String] + val keytabPath = provider.getConfigMap.get("keytabPath").asInstanceOf[String] + val krb5Path = provider.getConfigMap.get("krb5Path").asInstanceOf[String] val requestKuu = new RequestKerberosUrlUtils(principalName, keytabPath, krb5Path, false) val response = requestKuu.callRestUrl(rmWebAddress + "/ws/v1/cluster/" + url, principalName) @@ -325,8 +341,9 @@ class YarnResourceRequester extends ExternalResourceRequester with Logging { parse(EntityUtils.toString(httpResponse.getEntity())) } - def getAndUpdateActiveRmWebAddress(haAddress: String): String = { + def getAndUpdateActiveRmWebAddress(provider: ExternalResourceProvider): String = { // todo check if it will stuck for many requests + val haAddress = provider.getConfigMap.get("rmWebAddress").asInstanceOf[String] var activeAddress = rmAddressMap.get(haAddress) if (StringUtils.isBlank(activeAddress)) haAddress.intern().synchronized { if (StringUtils.isBlank(activeAddress)) { @@ -340,7 +357,7 @@ class YarnResourceRequester extends ExternalResourceRequester with Logging { .split(RMConfiguration.DEFAULT_YARN_RM_WEB_ADDRESS_DELIMITER.getValue) .foreach(address => { Utils.tryCatch { - val response = getResponseByUrl("info", address) + val response = getResponseByUrl("info", address, provider) response \ "clusterInfo" \ "haState" match { case state: JString => if (HASTATE_ACTIVE.equalsIgnoreCase(state.s)) { @@ -377,12 +394,10 @@ class YarnResourceRequester extends ExternalResourceRequester with Logging { override def reloadExternalResourceAddress( provider: ExternalResourceProvider ): java.lang.Boolean = { - if (null == provider) { - rmAddressMap.clear() - } else { + if (null != provider) { val rmWebHaAddress = provider.getConfigMap.get("rmWebAddress").asInstanceOf[String] rmAddressMap.remove(rmWebHaAddress) - getAndUpdateActiveRmWebAddress(rmWebHaAddress) + getAndUpdateActiveRmWebAddress(provider) } true } diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/restful/RMMonitorRest.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/restful/RMMonitorRest.scala index 8c59f1b17a..b50321e026 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/restful/RMMonitorRest.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/restful/RMMonitorRest.scala @@ -18,6 +18,7 @@ package org.apache.linkis.manager.rm.restful import org.apache.linkis.common.conf.Configuration +import org.apache.linkis.common.log.LogUtils import org.apache.linkis.common.utils.{Logging, Utils} import org.apache.linkis.manager.common.conf.RMConfiguration import org.apache.linkis.manager.common.entity.enumeration.NodeStatus @@ -25,7 +26,6 @@ import org.apache.linkis.manager.common.entity.node.EngineNode import org.apache.linkis.manager.common.entity.resource._ import org.apache.linkis.manager.common.errorcode.ManagerCommonErrorCodeSummary._ import org.apache.linkis.manager.common.exception.RMErrorException -import org.apache.linkis.manager.common.serializer.NodeResourceSerializer import org.apache.linkis.manager.common.utils.ResourceUtils import org.apache.linkis.manager.label.builder.CombinedLabelBuilder import org.apache.linkis.manager.label.builder.factory.LabelBuilderFactoryContext @@ -54,7 +54,6 @@ import org.apache.linkis.server.{toScalaBuffer, BDPJettyServerHelper, Message} import org.apache.linkis.server.security.SecurityFilter import org.apache.linkis.server.utils.ModuleUserUtils -import org.apache.commons.collections4.ListUtils import org.apache.commons.lang3.StringUtils import org.springframework.beans.factory.annotation.Autowired @@ -71,19 +70,18 @@ import scala.collection.mutable import scala.collection.mutable.ArrayBuffer import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.module.scala.DefaultScalaModule import com.github.pagehelper.page.PageMethod import com.google.common.collect.Lists -import io.swagger.annotations.{Api, ApiImplicitParams, ApiModel, ApiOperation} -import org.json4s.DefaultFormats -import org.json4s.jackson.Serialization.write +import io.swagger.annotations.{Api, ApiOperation} @RestController @Api(tags = Array("resource management")) @RequestMapping(path = Array("/linkisManager/rm")) class RMMonitorRest extends Logging { - implicit val formats = DefaultFormats + ResourceSerializer + NodeResourceSerializer val mapper = new ObjectMapper() + mapper.registerModule(DefaultScalaModule) private val dateFormatLocal = new ThreadLocal[SimpleDateFormat]() { override protected def initialValue = new SimpleDateFormat("EEE MMM dd HH:mm:ss z yyyy") @@ -126,8 +124,11 @@ class RMMonitorRest extends Logging { var COMBINED_USERCREATOR_ENGINETYPE: String = _ - def appendMessageData(message: Message, key: String, value: AnyRef): Message = - message.data(key, mapper.readTree(write(value))) + def appendMessageData(message: Message, key: String, value: AnyRef): Message = { + val result = mapper.writeValueAsString(value) + logger.info(s"appendMessageData result: $result") + message.data(key, mapper.readTree(result)) + } @ApiOperation(value = "getApplicationList", notes = "get applicationList") @RequestMapping(path = Array("applicationlist"), method = Array(RequestMethod.POST)) @@ -143,92 +144,10 @@ class RMMonitorRest extends Logging { val engineType = if (param.get("engineType") == null) null else param.get("engineType").asInstanceOf[String] val nodes = getEngineNodes(userName, true) - val creatorToApplicationList = new mutable.HashMap[String, mutable.HashMap[String, Any]] - nodes.foreach { node => - val userCreatorLabel = node.getLabels.asScala - .find(_.isInstanceOf[UserCreatorLabel]) - .get - .asInstanceOf[UserCreatorLabel] - val engineTypeLabel = node.getLabels.asScala - .find(_.isInstanceOf[EngineTypeLabel]) - .get - .asInstanceOf[EngineTypeLabel] - if (getUserCreator(userCreatorLabel).equals(userCreator)) { - if (engineType == null || getEngineType(engineTypeLabel).equals(engineType)) { - if (!creatorToApplicationList.contains(userCreatorLabel.getCreator)) { - val applicationList = new mutable.HashMap[String, Any] - applicationList.put("engineInstances", new mutable.ArrayBuffer[Any]) - applicationList.put("usedResource", Resource.initResource(ResourceType.LoadInstance)) - applicationList.put("maxResource", Resource.initResource(ResourceType.LoadInstance)) - applicationList.put("minResource", Resource.initResource(ResourceType.LoadInstance)) - applicationList.put("lockedResource", Resource.initResource(ResourceType.LoadInstance)) - creatorToApplicationList.put(userCreatorLabel.getCreator, applicationList) - } - val applicationList = creatorToApplicationList(userCreatorLabel.getCreator) - applicationList.put( - "usedResource", - (if (applicationList("usedResource") == null) { - Resource.initResource(ResourceType.LoadInstance) - } else { - applicationList("usedResource") - .asInstanceOf[Resource] - }) + node.getNodeResource.getUsedResource - ) - applicationList.put( - "maxResource", - (if (applicationList("maxResource") == null) { - Resource.initResource(ResourceType.LoadInstance) - } else { - applicationList("maxResource") - .asInstanceOf[Resource] - }) + node.getNodeResource.getMaxResource - ) - applicationList.put( - "minResource", - (if (applicationList("minResource") == null) { - Resource.initResource(ResourceType.LoadInstance) - } else { - applicationList("minResource") - .asInstanceOf[Resource] - }) + node.getNodeResource.getMinResource - ) - applicationList.put( - "lockedResource", - (if (applicationList("lockedResource") == null) { - Resource.initResource(ResourceType.LoadInstance) - } else { - applicationList("lockedResource") - .asInstanceOf[Resource] - }) + node.getNodeResource.getLockedResource - ) - val engineInstance = new mutable.HashMap[String, Any] - engineInstance.put("creator", userCreatorLabel.getCreator) - engineInstance.put("engineType", engineTypeLabel.getEngineType) - engineInstance.put("instance", node.getServiceInstance.getInstance) - engineInstance.put("label", engineTypeLabel.getStringValue) - node.setNodeResource( - ResourceUtils.convertTo(node.getNodeResource, ResourceType.LoadInstance) - ) - engineInstance.put("resource", node.getNodeResource) - if (node.getNodeStatus == null) { - engineInstance.put("status", "Busy") - } else { - engineInstance.put("status", node.getNodeStatus.toString) - } - engineInstance.put("startTime", dateFormatLocal.get().format(node.getStartTime)) - engineInstance.put("owner", node.getOwner) - applicationList("engineInstances") - .asInstanceOf[mutable.ArrayBuffer[Any]] - .append(engineInstance) - } - } - } - val applications = creatorToApplicationList.map { creatorEntry => - val application = new mutable.HashMap[String, Any] - application.put("creator", creatorEntry._1) - application.put("applicationList", creatorEntry._2) - application - } + + val creatorToApplicationList = getCreatorToApplicationList(userCreator, engineType, nodes) + + val applications = getApplications(creatorToApplicationList) appendMessageData(message, "applications", applications) message } @@ -338,112 +257,12 @@ class RMMonitorRest extends Logging { node.getLabels.asScala.find(_.isInstanceOf[EngineTypeLabel]).get != null }) } + val userCreatorEngineTypeResourceMap = - new mutable.HashMap[String, mutable.HashMap[String, NodeResource]] - nodes.foreach { node => - val userCreatorLabel = node.getLabels.asScala - .find(_.isInstanceOf[UserCreatorLabel]) - .get - .asInstanceOf[UserCreatorLabel] - val engineTypeLabel = node.getLabels.asScala - .find(_.isInstanceOf[EngineTypeLabel]) - .get - .asInstanceOf[EngineTypeLabel] - val userCreator = getUserCreator(userCreatorLabel) - if (!userCreatorEngineTypeResourceMap.contains(userCreator)) { - userCreatorEngineTypeResourceMap.put(userCreator, new mutable.HashMap[String, NodeResource]) - } - val engineTypeResourceMap = userCreatorEngineTypeResourceMap.get(userCreator).get - val engineType = getEngineType(engineTypeLabel) - if (!engineTypeResourceMap.contains(engineType)) { - val nodeResource = CommonNodeResource.initNodeResource(ResourceType.LoadInstance) - engineTypeResourceMap.put(engineType, nodeResource) - } - val resource = engineTypeResourceMap.get(engineType).get - resource.setUsedResource(node.getNodeResource.getUsedResource + resource.getUsedResource) - // combined label - val combinedLabel = - combinedLabelBuilder.build("", Lists.newArrayList(userCreatorLabel, engineTypeLabel)); - var labelResource = labelResourceService.getLabelResource(combinedLabel) - if (labelResource == null) { - resource.setLeftResource(node.getNodeResource.getMaxResource - resource.getUsedResource) - } else { - labelResource = ResourceUtils.convertTo(labelResource, ResourceType.LoadInstance) - resource.setUsedResource(labelResource.getUsedResource) - resource.setLockedResource(labelResource.getLockedResource) - resource.setLeftResource(labelResource.getLeftResource) - resource.setMaxResource(labelResource.getMaxResource) - } - resource.getLeftResource match { - case dResource: DriverAndYarnResource => - resource.setLeftResource(dResource.loadInstanceResource) - case _ => - } - } - val userCreatorEngineTypeResources = userCreatorEngineTypeResourceMap.map { userCreatorEntry => - val userCreatorEngineTypeResource = new mutable.HashMap[String, Any] - userCreatorEngineTypeResource.put("userCreator", userCreatorEntry._1) - var totalUsedMemory: Long = 0L - var totalUsedCores: Int = 0 - var totalUsedInstances = 0 - var totalLockedMemory: Long = 0L - var totalLockedCores: Int = 0 - var totalLockedInstances: Int = 0 - var totalMaxMemory: Long = 0L - var totalMaxCores: Int = 0 - var totalMaxInstances: Int = 0 - val engineTypeResources = userCreatorEntry._2.map { engineTypeEntry => - val engineTypeResource = new mutable.HashMap[String, Any] - engineTypeResource.put("engineType", engineTypeEntry._1) - val engineResource = engineTypeEntry._2 - val usedResource = engineResource.getUsedResource.asInstanceOf[LoadInstanceResource] - val lockedResource = engineResource.getLockedResource.asInstanceOf[LoadInstanceResource] - val maxResource = engineResource.getMaxResource.asInstanceOf[LoadInstanceResource] - val usedMemory = usedResource.memory - val usedCores = usedResource.cores - val usedInstances = usedResource.instances - totalUsedMemory += usedMemory - totalUsedCores += usedCores - totalUsedInstances += usedInstances - val lockedMemory = lockedResource.memory - val lockedCores = lockedResource.cores - val lockedInstances = lockedResource.instances - totalLockedMemory += lockedMemory - totalLockedCores += lockedCores - totalLockedInstances += lockedInstances - val maxMemory = maxResource.memory - val maxCores = maxResource.cores - val maxInstances = maxResource.instances - totalMaxMemory += maxMemory - totalMaxCores += maxCores - totalMaxInstances += maxInstances + getUserCreatorEngineTypeResourceMap(nodes) + + val userCreatorEngineTypeResources = getUserResources(userCreatorEngineTypeResourceMap) - val memoryPercent = - if (maxMemory > 0) (usedMemory + lockedMemory) / maxMemory.toDouble else 0 - val coresPercent = - if (maxCores > 0) (usedCores + lockedCores) / maxCores.toDouble else 0 - val instancePercent = - if (maxInstances > 0) (usedInstances + lockedInstances) / maxInstances.toDouble else 0 - val maxPercent = Math.max(Math.max(memoryPercent, coresPercent), instancePercent) - engineTypeResource.put("percent", maxPercent.formatted("%.2f")) - engineTypeResource - } - val totalMemoryPercent = - if (totalMaxMemory > 0) (totalUsedMemory + totalLockedMemory) / totalMaxMemory.toDouble - else 0 - val totalCoresPercent = - if (totalMaxCores > 0) (totalUsedCores + totalLockedCores) / totalMaxCores.toDouble - else 0 - val totalInstancePercent = - if (totalMaxInstances > 0) { - (totalUsedInstances + totalLockedInstances) / totalMaxInstances.toDouble - } else 0 - val totalPercent = - Math.max(Math.max(totalMemoryPercent, totalCoresPercent), totalInstancePercent) - userCreatorEngineTypeResource.put("engineTypes", engineTypeResources) - userCreatorEngineTypeResource.put("percent", totalPercent.formatted("%.2f")) - userCreatorEngineTypeResource - } appendMessageData(message, "userResources", userCreatorEngineTypeResources) message } @@ -500,6 +319,7 @@ class RMMonitorRest extends Logging { request: HttpServletRequest, @RequestBody param: util.Map[String, AnyRef] ): Message = { + ModuleUserUtils.getOperationUser(request, "getQueueResource") val message = Message.ok("") val yarnIdentifier = new YarnResourceIdentifier(param.get("queuename").asInstanceOf[String]) val clusterLabel = labelFactory.createLabel(classOf[ClusterLabel]) @@ -532,6 +352,9 @@ class RMMonitorRest extends Logging { "usedPercentage", Map("memory" -> usedMemoryPercentage, "cores" -> usedCPUPercentage) ) + queueInfo.put("maxApps", providedYarnResource.getMaxApps) + queueInfo.put("numActiveApps", providedYarnResource.getNumActiveApps) + queueInfo.put("numPendingApps", providedYarnResource.getNumPendingApps) appendMessageData(message, "queueInfo", queueInfo) case _ => Message.error("Failed to get queue resource") } @@ -780,4 +603,239 @@ class RMMonitorRest extends Logging { .groupBy(_.getOwner) } + private def getUserResources( + userCreatorEngineTypeResourceMap: mutable.HashMap[ + String, + mutable.HashMap[String, NodeResource] + ] + ) = { + + val userCreatorEngineTypeResources = userCreatorEngineTypeResourceMap.map { userCreatorEntry => + val userCreatorEngineTypeResource = new mutable.HashMap[String, Any] + userCreatorEngineTypeResource.put("userCreator", userCreatorEntry._1) + var totalUsedMemory: Long = 0L + var totalUsedCores: Int = 0 + var totalUsedInstances = 0 + var totalLockedMemory: Long = 0L + var totalLockedCores: Int = 0 + var totalLockedInstances: Int = 0 + var totalMaxMemory: Long = 0L + var totalMaxCores: Int = 0 + var totalMaxInstances: Int = 0 + val engineTypeResources = userCreatorEntry._2.map { engineTypeEntry => + val engineTypeResource = new mutable.HashMap[String, Any] + engineTypeResource.put("engineType", engineTypeEntry._1) + val engineResource = engineTypeEntry._2 + val usedResource = engineResource.getUsedResource.asInstanceOf[LoadInstanceResource] + val lockedResource = engineResource.getLockedResource.asInstanceOf[LoadInstanceResource] + val maxResource = engineResource.getMaxResource.asInstanceOf[LoadInstanceResource] + val usedMemory = usedResource.memory + val usedCores = usedResource.cores + val usedInstances = usedResource.instances + totalUsedMemory += usedMemory + totalUsedCores += usedCores + totalUsedInstances += usedInstances + val lockedMemory = lockedResource.memory + val lockedCores = lockedResource.cores + val lockedInstances = lockedResource.instances + totalLockedMemory += lockedMemory + totalLockedCores += lockedCores + totalLockedInstances += lockedInstances + val maxMemory = maxResource.memory + val maxCores = maxResource.cores + val maxInstances = maxResource.instances + totalMaxMemory += maxMemory + totalMaxCores += maxCores + totalMaxInstances += maxInstances + + val memoryPercent = + if (maxMemory > 0) (usedMemory + lockedMemory) / maxMemory.toDouble else 0 + val coresPercent = + if (maxCores > 0) (usedCores + lockedCores) / maxCores.toDouble else 0 + val instancePercent = + if (maxInstances > 0) (usedInstances + lockedInstances) / maxInstances.toDouble else 0 + val maxPercent = Math.max(Math.max(memoryPercent, coresPercent), instancePercent) + engineTypeResource.put("percent", maxPercent.formatted("%.2f")) + engineTypeResource + } + val totalMemoryPercent = + if (totalMaxMemory > 0) (totalUsedMemory + totalLockedMemory) / totalMaxMemory.toDouble + else 0 + val totalCoresPercent = + if (totalMaxCores > 0) (totalUsedCores + totalLockedCores) / totalMaxCores.toDouble + else 0 + val totalInstancePercent = + if (totalMaxInstances > 0) { + (totalUsedInstances + totalLockedInstances) / totalMaxInstances.toDouble + } else 0 + val totalPercent = + Math.max(Math.max(totalMemoryPercent, totalCoresPercent), totalInstancePercent) + userCreatorEngineTypeResource.put("engineTypes", engineTypeResources) + userCreatorEngineTypeResource.put("percent", totalPercent.formatted("%.2f")) + userCreatorEngineTypeResource + } + userCreatorEngineTypeResources + } + + private def getUserCreatorEngineTypeResourceMap(nodes: Array[EngineNode]) = { + val userCreatorEngineTypeResourceMap = + new mutable.HashMap[String, mutable.HashMap[String, NodeResource]] + + for (node <- nodes) { + val userCreatorLabel = node.getLabels.asScala + .find(_.isInstanceOf[UserCreatorLabel]) + .get + .asInstanceOf[UserCreatorLabel] + val engineTypeLabel = node.getLabels.asScala + .find(_.isInstanceOf[EngineTypeLabel]) + .get + .asInstanceOf[EngineTypeLabel] + val userCreator = getUserCreator(userCreatorLabel) + + if (!userCreatorEngineTypeResourceMap.contains(userCreator)) { + userCreatorEngineTypeResourceMap.put(userCreator, new mutable.HashMap[String, NodeResource]) + } + val engineTypeResourceMap = userCreatorEngineTypeResourceMap.get(userCreator).get + val engineType = getEngineType(engineTypeLabel) + if (!engineTypeResourceMap.contains(engineType)) { + val nodeResource = CommonNodeResource.initNodeResource(ResourceType.LoadInstance) + engineTypeResourceMap.put(engineType, nodeResource) + } + val resource = engineTypeResourceMap.get(engineType).get + resource.setUsedResource(node.getNodeResource.getUsedResource + resource.getUsedResource) + // combined label + val combinedLabel = + combinedLabelBuilder.build("", Lists.newArrayList(userCreatorLabel, engineTypeLabel)); + var labelResource = labelResourceService.getLabelResource(combinedLabel) + if (labelResource == null) { + resource.setLeftResource(node.getNodeResource.getMaxResource - resource.getUsedResource) + } else { + labelResource = ResourceUtils.convertTo(labelResource, ResourceType.LoadInstance) + resource.setUsedResource(labelResource.getUsedResource) + resource.setLockedResource(labelResource.getLockedResource) + resource.setLeftResource(labelResource.getLeftResource) + resource.setMaxResource(labelResource.getMaxResource) + } + resource.getLeftResource match { + case dResource: DriverAndYarnResource => + resource.setLeftResource(dResource.loadInstanceResource) + case _ => + } + } + + userCreatorEngineTypeResourceMap + } + + private def getCreatorToApplicationList( + userCreator: String, + engineType: String, + nodes: Array[EngineNode] + ) = { + val creatorToApplicationList = new util.HashMap[String, util.HashMap[String, Any]] + nodes.foreach { node => + val userCreatorLabel = node.getLabels.asScala + .find(_.isInstanceOf[UserCreatorLabel]) + .get + .asInstanceOf[UserCreatorLabel] + val engineTypeLabel = node.getLabels.asScala + .find(_.isInstanceOf[EngineTypeLabel]) + .get + .asInstanceOf[EngineTypeLabel] + if (getUserCreator(userCreatorLabel).equals(userCreator)) { + if (engineType == null || getEngineType(engineTypeLabel).equals(engineType)) { + if (!creatorToApplicationList.containsKey(userCreatorLabel.getCreator)) { + val applicationList = new util.HashMap[String, Any] + applicationList.put("engineInstances", new util.ArrayList[Any]) + applicationList.put("usedResource", Resource.initResource(ResourceType.LoadInstance)) + applicationList.put("maxResource", Resource.initResource(ResourceType.LoadInstance)) + applicationList.put("minResource", Resource.initResource(ResourceType.LoadInstance)) + applicationList.put("lockedResource", Resource.initResource(ResourceType.LoadInstance)) + creatorToApplicationList.put(userCreatorLabel.getCreator, applicationList) + } + val applicationList = creatorToApplicationList.get(userCreatorLabel.getCreator) + applicationList.put( + "usedResource", + (if (applicationList.get("usedResource") == null) { + Resource.initResource(ResourceType.LoadInstance) + } else { + applicationList + .get("usedResource") + .asInstanceOf[Resource] + }) + node.getNodeResource.getUsedResource + ) + applicationList.put( + "maxResource", + (if (applicationList.get("maxResource") == null) { + Resource.initResource(ResourceType.LoadInstance) + } else { + applicationList + .get("maxResource") + .asInstanceOf[Resource] + }) + node.getNodeResource.getMaxResource + ) + applicationList.put( + "minResource", + (if (applicationList.get("minResource") == null) { + Resource.initResource(ResourceType.LoadInstance) + } else { + applicationList + .get("minResource") + .asInstanceOf[Resource] + }) + node.getNodeResource.getMinResource + ) + applicationList.put( + "lockedResource", + (if (applicationList.get("lockedResource") == null) { + Resource.initResource(ResourceType.LoadInstance) + } else { + applicationList + .get("lockedResource") + .asInstanceOf[Resource] + }) + node.getNodeResource.getLockedResource + ) + val engineInstance = new mutable.HashMap[String, Any] + engineInstance.put("creator", userCreatorLabel.getCreator) + engineInstance.put("engineType", engineTypeLabel.getEngineType) + engineInstance.put("instance", node.getServiceInstance.getInstance) + engineInstance.put("label", engineTypeLabel.getStringValue) + node.setNodeResource( + ResourceUtils.convertTo(node.getNodeResource, ResourceType.LoadInstance) + ) + engineInstance.put("resource", node.getNodeResource) + if (node.getNodeStatus == null) { + engineInstance.put("status", "Busy") + } else { + engineInstance.put("status", node.getNodeStatus.toString) + } + engineInstance.put( + "st" + + "artTime", + dateFormatLocal.get().format(node.getStartTime) + ) + engineInstance.put("owner", node.getOwner) + applicationList + .get("engineInstances") + .asInstanceOf[util.ArrayList[Any]] + .add(engineInstance) + } + } + } + creatorToApplicationList + } + + private def getApplications( + creatorToApplicationList: util.HashMap[String, util.HashMap[String, Any]] + ) = { + val applications = new util.ArrayList[util.HashMap[String, Any]]() + val iterator = creatorToApplicationList.entrySet().iterator(); + while (iterator.hasNext) { + val entry = iterator.next() + val application = new util.HashMap[String, Any] + application.put("creator", entry.getKey) + application.put("applicationList", entry.getValue) + applications.add(application) + } + applications + } + } diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/RequestResourceService.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/RequestResourceService.scala index 70fb1dd9f5..d990fd7045 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/RequestResourceService.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/RequestResourceService.scala @@ -22,6 +22,7 @@ import org.apache.linkis.manager.common.constant.RMConstant import org.apache.linkis.manager.common.entity.resource._ import org.apache.linkis.manager.common.errorcode.ManagerCommonErrorCodeSummary._ import org.apache.linkis.manager.common.exception.RMWarnException +import org.apache.linkis.manager.common.protocol.engine.{EngineAskRequest, EngineCreateRequest} import org.apache.linkis.manager.label.entity.em.EMInstanceLabel import org.apache.linkis.manager.rm.domain.RMLabelContainer import org.apache.linkis.manager.rm.exception.RMErrorCode @@ -35,7 +36,11 @@ abstract class RequestResourceService(labelResourceService: LabelResourceService val enableRequest = RMUtils.RM_REQUEST_ENABLE.getValue - def canRequest(labelContainer: RMLabelContainer, resource: NodeResource): Boolean = { + def canRequest( + labelContainer: RMLabelContainer, + resource: NodeResource, + engineCreateRequest: EngineCreateRequest + ): Boolean = { labelContainer.getCurrentLabel match { case emInstanceLabel: EMInstanceLabel => @@ -131,7 +136,10 @@ abstract class RequestResourceService(labelResourceService: LabelResourceService labelAvailableResource, labelResource.getMaxResource ) - throw new RMWarnException(notEnoughMessage._1, notEnoughMessage._2) + throw new RMWarnException( + notEnoughMessage._1, + notEnoughMessage._2 + s"ECM Instance:${emInstanceLabel.getInstance()}" + ) } logger.debug(s"Passed check: resource[${requestResource}] want to use em ${emInstanceLabel .getInstance()} available resource[${labelAvailableResource}]") diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/ResourceManager.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/ResourceManager.scala index 78e414d434..62babebf8e 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/ResourceManager.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/ResourceManager.scala @@ -20,6 +20,7 @@ package org.apache.linkis.manager.rm.service import org.apache.linkis.common.ServiceInstance import org.apache.linkis.manager.common.entity.node.EngineNode import org.apache.linkis.manager.common.entity.resource.NodeResource +import org.apache.linkis.manager.common.protocol.engine.{EngineAskRequest, EngineCreateRequest} import org.apache.linkis.manager.label.entity.Label import org.apache.linkis.manager.rm.{ResourceInfo, ResultResource} @@ -46,7 +47,11 @@ abstract class ResourceManager { * @param resource * @return */ - def requestResource(labels: util.List[Label[_]], resource: NodeResource): ResultResource + def requestResource( + labels: util.List[Label[_]], + resource: NodeResource, + engineCreateRequest: EngineCreateRequest + ): ResultResource /** * Request resources and wait for a certain amount of time until the requested resource is met @@ -60,6 +65,7 @@ abstract class ResourceManager { def requestResource( labels: util.List[Label[_]], resource: NodeResource, + engineCreateRequest: EngineCreateRequest, wait: Long ): ResultResource diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/impl/DefaultResourceManager.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/impl/DefaultResourceManager.scala index f1284268b3..2b1702069d 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/impl/DefaultResourceManager.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/impl/DefaultResourceManager.scala @@ -33,6 +33,7 @@ import org.apache.linkis.manager.common.entity.persistence.{ import org.apache.linkis.manager.common.entity.resource._ import org.apache.linkis.manager.common.errorcode.ManagerCommonErrorCodeSummary import org.apache.linkis.manager.common.exception.{RMErrorException, RMWarnException} +import org.apache.linkis.manager.common.protocol.engine.{EngineAskRequest, EngineCreateRequest} import org.apache.linkis.manager.common.utils.{ManagerUtils, ResourceUtils} import org.apache.linkis.manager.label.builder.factory.LabelBuilderFactoryContext import org.apache.linkis.manager.label.constant.LabelKeyConstant @@ -52,6 +53,7 @@ import org.apache.linkis.manager.rm.{ ResourceInfo, ResultResource } +import org.apache.linkis.manager.rm.domain.RMLabelContainer import org.apache.linkis.manager.rm.entity.{LabelResourceMapping, ResourceOperationType} import org.apache.linkis.manager.rm.entity.ResourceOperationType.{LOCK, USED} import org.apache.linkis.manager.rm.exception.{RMErrorCode, RMLockFailedRetryException} @@ -152,22 +154,40 @@ class DefaultResourceManager extends ResourceManager with Logging with Initializ eMInstanceLabel.setInstance(serviceInstance.getInstance) val emResource = labelResourceService.getLabelResource(eMInstanceLabel) + var registerResourceFlag = true if (emResource != null) { - logger.warn(s"${serviceInstance} has been registered, now update resource.") - if (!emResource.getResourceType.equals(resource.getResourceType)) { - throw new RMErrorException( - RMErrorCode.LABEL_DUPLICATED.getErrorCode, - MessageFormat.format( - RMErrorCode.LABEL_DUPLICATED.getErrorDesc, - serviceInstance, - emResource.getResourceType, - resource.getResourceType - ) + registerResourceFlag = false + logger.warn(s"${serviceInstance} has been registered, resource is ${emResource}.") + val leftResource = emResource.getLeftResource + if (leftResource != null && Resource.getZeroResource(leftResource) > leftResource) { + logger.warn( + s"${serviceInstance} has been registered, but left Resource <0 need to register resource." + ) + registerResourceFlag = true + } + val usedResource = emResource.getLockedResource + emResource.getUsedResource + if (usedResource > emResource.getMaxResource) { + logger.warn( + s"${serviceInstance} has been registered, but usedResource > MaxResource need to register resource." + ) + registerResourceFlag = true + } + + if (!(resource.getMaxResource == emResource.getMaxResource)) { + logger.warn( + s"${serviceInstance} has been registered, but inconsistent newly registered resources need to register resource." ) + registerResourceFlag = true } } + + if (!registerResourceFlag) { + logger.warn(s"${serviceInstance} has been registered, skip register resource.") + return + } val lock = tryLockOneLabel(eMInstanceLabel, -1, Utils.getJvmUser) try { + labelResourceService.removeResourceByLabel(eMInstanceLabel) labelResourceService.setLabelResource( eMInstanceLabel, resource, @@ -190,8 +210,8 @@ class DefaultResourceManager extends ResourceManager with Logging with Initializ } /** - * The registration method is mainly used to notify all RM nodes (including the node), and the - * instance is offline. 该注册方法，主要是用于通知所有的RM节点（包括本节点），下线该实例 + * The registration method is mainly used to notify all RM nodes , and the instance is offline. + * 该注册方法，主要是用于通知所有的RM节点（包括本节点），下线该实例 */ override def unregister(serviceInstance: ServiceInstance): Unit = { @@ -199,7 +219,6 @@ class DefaultResourceManager extends ResourceManager with Logging with Initializ LabelBuilderFactoryContext.getLabelBuilderFactory.createLabel(classOf[EMInstanceLabel]) eMInstanceLabel.setServiceName(serviceInstance.getApplicationName) eMInstanceLabel.setInstance(serviceInstance.getInstance) - val ecNodes = nodeManagerPersistence.getEngineNodeByEM(serviceInstance).asScala val lock = tryLockOneLabel(eMInstanceLabel, -1, Utils.getJvmUser) try { labelResourceService.removeResourceByLabel(eMInstanceLabel) @@ -217,13 +236,6 @@ class DefaultResourceManager extends ResourceManager with Logging with Initializ resourceLockService.unLock(lock) logger.info(s"Finished to clear ecm resource:${serviceInstance}") } - ecNodes.foreach { engineNode => - Utils.tryAndWarn { - engineNode.setLabels(nodeLabelService.getNodeLabels(engineNode.getServiceInstance)) - engineNode.setNodeStatus(NodeStatus.Failed) - engineStopService.engineConnInfoClear(engineNode) - } - } logger.info(s"Finished to clear ec for ecm ${serviceInstance}") } @@ -236,9 +248,10 @@ class DefaultResourceManager extends ResourceManager with Logging with Initializ */ override def requestResource( labels: util.List[Label[_]], - resource: NodeResource + resource: NodeResource, + engineCreateRequest: EngineCreateRequest ): ResultResource = { - requestResource(labels, resource, -1) + requestResource(labels, resource, engineCreateRequest, -1) } /** @@ -253,6 +266,7 @@ class DefaultResourceManager extends ResourceManager with Logging with Initializ override def requestResource( labels: util.List[Label[_]], resource: NodeResource, + engineCreateRequest: EngineCreateRequest, wait: Long ): ResultResource = { val labelContainer = labelResourceService.enrichLabels(labels) @@ -268,7 +282,7 @@ class DefaultResourceManager extends ResourceManager with Logging with Initializ // check ecm resource if not enough return Utils.tryCatch { labelContainer.setCurrentLabel(emInstanceLabel) - if (!requestResourceService.canRequest(labelContainer, resource)) { + if (!requestResourceService.canRequest(labelContainer, resource, engineCreateRequest)) { return NotEnoughResource(s"Labels:${emInstanceLabel.getStringValue} not enough resource") } } { @@ -286,7 +300,7 @@ class DefaultResourceManager extends ResourceManager with Logging with Initializ ) Utils.tryCatch { labelContainer.setCurrentLabel(userCreatorEngineTypeLabel) - if (!requestResourceService.canRequest(labelContainer, resource)) { + if (!requestResourceService.canRequest(labelContainer, resource, engineCreateRequest)) { return NotEnoughResource( s"Labels:${userCreatorEngineTypeLabel.getStringValue} not enough resource" ) @@ -320,22 +334,19 @@ class DefaultResourceManager extends ResourceManager with Logging with Initializ } { persistenceLocks.foreach(resourceLockService.unLock) } - // record engine locked resource - val tickedId = UUID.randomUUID().toString - resourceLogService.recordUserResourceAction( - labelContainer, - tickedId, - ChangeType.ENGINE_REQUEST, - resource.getLockedResource - ) + + // add ec node + val tickedId = RMUtils.getECTicketID val emNode = new AMEMNode emNode.setServiceInstance(labelContainer.getEMInstanceLabel.getServiceInstance) val engineNode = new AMEngineNode engineNode.setEMNode(emNode) engineNode.setServiceInstance(ServiceInstance(labelContainer.getEngineServiceName, tickedId)) engineNode.setNodeResource(resource) + engineNode.setTicketId(tickedId) nodeManagerPersistence.addEngineNode(engineNode) + // add labels val engineInstanceLabel = LabelBuilderFactoryContext.getLabelBuilderFactory.createLabel(classOf[EngineInstanceLabel]) engineInstanceLabel.setServiceName(labelContainer.getEngineServiceName) @@ -343,11 +354,20 @@ class DefaultResourceManager extends ResourceManager with Logging with Initializ nodeLabelService.addLabelToNode(engineNode.getServiceInstance, engineInstanceLabel) + // add ec resource labelResourceService.setEngineConnLabelResource( engineInstanceLabel, resource, labelContainer.getCombinedUserCreatorEngineTypeLabel.getStringValue ) + // record engine locked resource + labelContainer.getLabels.add(engineInstanceLabel) + resourceLogService.recordUserResourceAction( + labelContainer, + tickedId, + ChangeType.ENGINE_REQUEST, + resource.getLockedResource + ) val persistenceLabel = labelFactory.convertLabel(engineInstanceLabel, classOf[PersistenceLabel]) val persistenceEngineLabel = labelManagerPersistence.getLabelByKeyValue( @@ -710,7 +730,7 @@ class DefaultResourceManager extends ResourceManager with Logging with Initializ var heartbeatMsgMetrics = "" Utils.tryAndWarn { val oldMetrics = nodeMetricManagerPersistence.getNodeMetrics(ecNode) - if (StringUtils.isNotBlank(oldMetrics.getHeartBeatMsg)) { + if (oldMetrics != null && StringUtils.isNotBlank(oldMetrics.getHeartBeatMsg)) { heartbeatMsgMetrics = oldMetrics.getHeartBeatMsg } } diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/impl/DriverAndYarnReqResourceService.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/impl/DriverAndYarnReqResourceService.scala index e82ff8383c..b934c4123d 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/impl/DriverAndYarnReqResourceService.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/impl/DriverAndYarnReqResourceService.scala @@ -17,16 +17,20 @@ package org.apache.linkis.manager.rm.service.impl +import org.apache.linkis.manager.am.conf.AMConfiguration import org.apache.linkis.manager.common.constant.RMConstant import org.apache.linkis.manager.common.entity.resource._ import org.apache.linkis.manager.common.entity.resource.ResourceType.DriverAndYarn import org.apache.linkis.manager.common.exception.RMWarnException +import org.apache.linkis.manager.common.protocol.engine.EngineCreateRequest import org.apache.linkis.manager.rm.domain.RMLabelContainer import org.apache.linkis.manager.rm.exception.RMErrorCode import org.apache.linkis.manager.rm.external.service.ExternalResourceService import org.apache.linkis.manager.rm.external.yarn.YarnResourceIdentifier import org.apache.linkis.manager.rm.service.{LabelResourceService, RequestResourceService} -import org.apache.linkis.manager.rm.utils.RMUtils +import org.apache.linkis.manager.rm.utils.{AcrossClusterRulesJudgeUtils, RMUtils} + +import org.apache.commons.lang3.StringUtils import org.json4s.DefaultFormats @@ -39,10 +43,15 @@ class DriverAndYarnReqResourceService( override val resourceType: ResourceType = DriverAndYarn - override def canRequest(labelContainer: RMLabelContainer, resource: NodeResource): Boolean = { - if (!super.canRequest(labelContainer, resource)) { + override def canRequest( + labelContainer: RMLabelContainer, + resource: NodeResource, + engineCreateRequest: EngineCreateRequest + ): Boolean = { + if (!super.canRequest(labelContainer, resource, engineCreateRequest)) { return false } + val requestedDriverAndYarnResource = resource.getMaxResource.asInstanceOf[DriverAndYarnResource] val requestedYarnResource = requestedDriverAndYarnResource.yarnResource @@ -65,7 +74,58 @@ class DriverAndYarnReqResourceService( val notEnoughMessage = generateQueueNotEnoughMessage(requestedYarnResource, queueLeftResource, maxCapacity) throw new RMWarnException(notEnoughMessage._1, notEnoughMessage._2) - } else true + } + + if (engineCreateRequest.getProperties != null) { + val user = labelContainer.getUserCreatorLabel.getUser + val creator = labelContainer.getUserCreatorLabel.getCreator + val properties = engineCreateRequest.getProperties + val acrossClusterTask = properties.getOrDefault(AMConfiguration.ACROSS_CLUSTER_TASK, "false") + val CPUThreshold = properties.get(AMConfiguration.ACROSS_CLUSTER_CPU_THRESHOLD) + val MemoryThreshold = properties.get(AMConfiguration.ACROSS_CLUSTER_MEMORY_THRESHOLD) + val CPUPercentageThreshold = + properties.get(AMConfiguration.ACROSS_CLUSTER_CPU_PERCENTAGE_THRESHOLD) + val MemoryPercentageThreshold = + properties.get(AMConfiguration.ACROSS_CLUSTER_MEMORY_PERCENTAGE_THRESHOLD) + + if ( + StringUtils.isNotBlank(acrossClusterTask) && acrossClusterTask.toBoolean && StringUtils + .isNotBlank(CPUThreshold) && StringUtils + .isNotBlank(MemoryThreshold) + && StringUtils + .isNotBlank(CPUPercentageThreshold) && StringUtils.isNotBlank(MemoryPercentageThreshold) + ) { + + logger.info( + s"user: $user, creator: $creator task enter cross cluster resource judgment, " + + s"CPUThreshold: $CPUThreshold, MemoryThreshold: $MemoryThreshold," + + s"CPUPercentageThreshold: $CPUPercentageThreshold, MemoryPercentageThreshold: $MemoryPercentageThreshold" + ) + try { + AcrossClusterRulesJudgeUtils.acrossClusterRuleCheck( + queueLeftResource.asInstanceOf[YarnResource], + usedCapacity.asInstanceOf[YarnResource], + maxCapacity.asInstanceOf[YarnResource], + CPUThreshold.toInt, + MemoryThreshold.toInt, + CPUPercentageThreshold.toDouble, + MemoryPercentageThreshold.toDouble + ) + } catch { + case ex: Exception => + throw new RMWarnException( + RMErrorCode.ACROSS_CLUSTER_RULE_FAILED.getErrorCode, + ex.getMessage + ) + } + + logger.info(s"user: $user, creator: $creator task meet the threshold rule") + } else { + logger.info(s"user: $user, creator: $creator task skip cross cluster resource judgment") + } + } + + true } def generateQueueNotEnoughMessage( @@ -86,7 +146,8 @@ class DriverAndYarnReqResourceService( RMConstant.CPU_UNIT, yarn.queueCores, yarnAvailable.queueCores, - maxYarn.queueCores + maxYarn.queueCores, + yarn.queueName ) ) } else if (yarn.queueMemory > yarnAvailable.queueMemory) { @@ -98,7 +159,8 @@ class DriverAndYarnReqResourceService( RMConstant.MEMORY_UNIT_BYTE, yarn.queueMemory, yarnAvailable.queueMemory, - maxYarn.queueMemory + maxYarn.queueMemory, + yarn.queueName ) ) } else { @@ -110,7 +172,8 @@ class DriverAndYarnReqResourceService( RMConstant.INSTANCE_UNIT, yarn.queueInstances, yarnAvailable.queueInstances, - maxYarn.queueInstances + maxYarn.queueInstances, + yarn.queueName ) ) } diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/impl/ResourceLogService.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/impl/ResourceLogService.scala index e332854bc2..2d67edb9a7 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/impl/ResourceLogService.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/impl/ResourceLogService.scala @@ -25,7 +25,11 @@ import org.apache.linkis.manager.common.entity.resource.Resource import org.apache.linkis.manager.dao.ECResourceRecordMapper import org.apache.linkis.manager.label.entity.CombinedLabel import org.apache.linkis.manager.label.entity.em.EMInstanceLabel -import org.apache.linkis.manager.label.entity.engine.EngineInstanceLabel +import org.apache.linkis.manager.label.entity.engine.{ + EngineInstanceLabel, + EngineTypeLabel, + UserCreatorLabel +} import org.apache.linkis.manager.rm.domain.RMLabelContainer import org.apache.linkis.manager.rm.utils.RMUtils @@ -150,7 +154,11 @@ class ResourceLogService extends Logging { if (null == userCreatorEngineType) return var ecResourceInfoRecord = ecResourceRecordMapper.getECResourceInfoRecord(ticketId) if (ecResourceInfoRecord == null) { - val logDirSuffix = getECLogDirSuffix(labelContainer, ticketId) + val logDirSuffix = ECPathUtils.getECLogDirSuffix( + labelContainer.getEngineTypeLabel, + labelContainer.getUserCreatorLabel, + ticketId + ) val user = if (null != labelContainer.getUserCreatorLabel) labelContainer.getUserCreatorLabel.getUser else "" @@ -200,20 +208,6 @@ class ResourceLogService extends Logging { ecResourceRecordMapper.updateECResourceInfoRecord(ecResourceInfoRecord) } - def getECLogDirSuffix(labelContainer: RMLabelContainer, ticketId: String): String = { - val engineTypeLabel = labelContainer.getEngineTypeLabel - val userCreatorLabel = labelContainer.getUserCreatorLabel - if (null == engineTypeLabel || null == userCreatorLabel) { - return "" - } - val suffix = ECPathUtils.getECWOrkDirPathSuffix( - userCreatorLabel.getUser, - ticketId, - engineTypeLabel.getEngineType - ) - suffix + File.separator + "logs" - } - } object ChangeType { diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/utils/AcrossClusterRulesJudgeUtils.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/utils/AcrossClusterRulesJudgeUtils.scala new file mode 100644 index 0000000000..805785299d --- /dev/null +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/utils/AcrossClusterRulesJudgeUtils.scala @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.rm.utils + +import org.apache.linkis.common.utils.Logging +import org.apache.linkis.manager.common.entity.resource.YarnResource +import org.apache.linkis.manager.common.exception.RMWarnException +import org.apache.linkis.manager.rm.exception.RMErrorCode + +object AcrossClusterRulesJudgeUtils extends Logging { + + def acrossClusterRuleCheck( + leftResource: YarnResource, + usedResource: YarnResource, + maxResource: YarnResource, + leftCPUThreshold: Int, + leftMemoryThreshold: Int, + CPUPercentageThreshold: Double, + MemoryPercentageThreshold: Double + ): Unit = { + if (leftResource != null && usedResource != null && maxResource != null) { + val leftQueueMemory = leftResource.queueMemory / Math.pow(1024, 3).toLong + + if (leftResource.queueCores > leftCPUThreshold && leftQueueMemory > leftMemoryThreshold) { + val usedCPUPercentage = + usedResource.queueCores.asInstanceOf[Double] / maxResource.queueCores + .asInstanceOf[Double] + val usedMemoryPercentage = usedResource.queueMemory + .asInstanceOf[Double] / maxResource.queueMemory.asInstanceOf[Double] + + if ( + usedCPUPercentage < CPUPercentageThreshold && usedMemoryPercentage < MemoryPercentageThreshold + ) { + return + } else { + throw new RMWarnException( + RMErrorCode.ACROSS_CLUSTER_RULE_FAILED.getErrorCode, + s"usedCPUPercentage: $usedCPUPercentage, CPUPercentageThreshold: $CPUPercentageThreshold" + + s"usedMemoryPercentage: $usedMemoryPercentage, MemoryPercentageThreshold: $MemoryPercentageThreshold" + ) + } + } else { + throw new RMWarnException( + RMErrorCode.ACROSS_CLUSTER_RULE_FAILED.getErrorCode, + s"leftResource.queueCores: ${leftResource.queueCores}, leftCPUThreshold: $leftCPUThreshold," + + s"leftQueueMemory: $leftQueueMemory, leftMemoryThreshold: $leftMemoryThreshold" + ) + } + } + } + +} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/utils/RMUtils.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/utils/RMUtils.scala index ca5f420693..2b74c31079 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/utils/RMUtils.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/utils/RMUtils.scala @@ -28,7 +28,10 @@ import org.apache.linkis.manager.rm.conf.ResourceStatus import org.apache.linkis.manager.rm.restful.vo.UserResourceVo import org.apache.linkis.server.BDPJettyServerHelper +import org.apache.commons.lang3.StringUtils + import java.util +import java.util.UUID import scala.collection.JavaConverters.asScalaBufferConverter @@ -213,7 +216,8 @@ object RMUtils extends Logging { unitType: String, requestResource: Any, availableResource: Any, - maxResource: Any + maxResource: Any, + queueName: String = "" ): String = { def dealMemory(resourceType: String, unitType: String, resource: Any): String = { @@ -222,7 +226,7 @@ object RMUtils extends Logging { if (logger.isDebugEnabled()) { logger.debug(s"Will change ${resource.toString} from ${unitType} to GB") } - ByteTimeUtils.byteStringAsGb(resource.toString + "b").toString + "GB" + ByteTimeUtils.negativeByteStringAsGb(resource.toString + "b").toString + "GB" } { case e: Exception => logger.error(s"Cannot convert ${resource} to Gb, " + e.getMessage) resource.toString + unitType @@ -241,7 +245,13 @@ object RMUtils extends Logging { val maxMsg = if (null == maxResource) "null" + unitType else dealMemory(resourceType, unitType, maxResource.toString) - s" user ${resourceType}, requestResource : ${reqMsg} > availableResource : ${availMsg}, maxResource : ${maxMsg}." + if (StringUtils.isEmpty(queueName)) { + s" use ${resourceType}, requestResource : ${reqMsg} > availableResource : ${availMsg}, maxResource : ${maxMsg}." + } else { + s" use ${resourceType}, requestResource : ${reqMsg} > availableResource : ${availMsg}, maxResource : ${maxMsg}, queueName : ${queueName}." + } } + def getECTicketID: String = UUID.randomUUID().toString + } diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/test/java/org/apache/linkis/manager/am/util/ECResourceInfoUtilsTest.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/test/java/org/apache/linkis/manager/am/util/ECResourceInfoUtilsTest.java new file mode 100644 index 0000000000..1d1ce10cfd --- /dev/null +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/test/java/org/apache/linkis/manager/am/util/ECResourceInfoUtilsTest.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.am.util; + +import org.apache.linkis.common.utils.ByteTimeUtils; +import org.apache.linkis.manager.am.vo.ResourceVo; +import org.apache.linkis.manager.common.entity.persistence.ECResourceInfoRecord; +import org.apache.linkis.server.BDPJettyServerHelper; + +import java.util.HashMap; +import java.util.Map; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +/** ECResourceInfoUtilsTest */ +public class ECResourceInfoUtilsTest { + + @Test + public void testGetStringToMap() throws Exception { + ECResourceInfoRecord info = new ECResourceInfoRecord(); + info.setLabelValue("hadoop-LINKISCLI,spark-2.4.3"); + String str = + "{\"driver\":{\"instance\":1,\"memory\":\"3.0 GB\",\"cpu\":1}, \"yarn\":{\"queueName\":\"dws\",\"queueMemory\":\"2.0 GB\", \"queueCpu\":2, \"instance\":0}} "; + Map map = BDPJettyServerHelper.gson().fromJson(str, new HashMap<>().getClass()); + ResourceVo resourceVO = ECResourceInfoUtils.getStringToMap(str, info); + Map diverMap = (Map) map.get("driver"); + Assertions.assertEquals( + resourceVO.getInstance(), ((Double) diverMap.get("instance")).intValue()); + Assertions.assertEquals(resourceVO.getInstance(), 1); + Assertions.assertEquals(resourceVO.getCores(), ((Double) diverMap.get("cpu")).intValue()); + Assertions.assertEquals(resourceVO.getCores(), 1); + Assertions.assertEquals( + resourceVO.getMemory(), + ByteTimeUtils.byteStringAsBytes(String.valueOf(diverMap.getOrDefault("memory", "0k")))); + } +} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/interactor/job/subtype/LinkisSubmitSubTypeTest.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/test/scala/org/apache/linkis/manager/label/conf/LabelManagerConfTest.scala similarity index 66% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/interactor/job/subtype/LinkisSubmitSubTypeTest.java rename to linkis-computation-governance/linkis-manager/linkis-application-manager/src/test/scala/org/apache/linkis/manager/label/conf/LabelManagerConfTest.scala index f05d59700e..836278336b 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/interactor/job/subtype/LinkisSubmitSubTypeTest.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/test/scala/org/apache/linkis/manager/label/conf/LabelManagerConfTest.scala @@ -15,19 +15,19 @@ * limitations under the License. */ -package org.apache.linkis.cli.application.interactor.job.subtype; +package org.apache.linkis.manager.label.conf -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Test -public class LinkisSubmitSubTypeTest { +class LabelManagerConfTest { - @Test - @DisplayName("enumTest") - public void enumTest() { - - String submitName = LinkisSubmitSubType.SUBMIT.getName(); - Assertions.assertEquals("submit", submitName); + @Test def testRandomFiltering(): Unit = { + var label = "tenant" + assert(LabelManagerConf.LONG_LIVED_LABEL.contains(label)) + label = "yarnCluster" + assert(LabelManagerConf.LONG_LIVED_LABEL.contains(label)) + label = "test" + assert(!LabelManagerConf.LONG_LIVED_LABEL.contains(label)) } + } diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/test/scala/org/apache/linkis/manager/rm/utils/RMUtilsTest.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/test/scala/org/apache/linkis/manager/rm/utils/RMUtilsTest.scala new file mode 100644 index 0000000000..35039a394f --- /dev/null +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/test/scala/org/apache/linkis/manager/rm/utils/RMUtilsTest.scala @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.rm.utils + +import org.junit.jupiter.api.Test + +class RMUtilsTest { + + @Test def getResourceInfoMsg(): Unit = { + val resourceType: String = "Memory" + val unitType: String = "bytes" + val requestResource: Any = "644245094400" // 600G + val availableResource: Any = "-2147483648" // -2G + val maxResource: Any = "20454781747200" // 19050G + val result = RMUtils.getResourceInfoMsg( + resourceType, + unitType, + requestResource, + availableResource, + maxResource + ) + assert( + " use Memory, requestResource : 600GB > availableResource : -2GB, maxResource : 19050GB." + .equals(result) + ) + } + +} diff --git a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/constant/LabelKeyConstant.java b/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/constant/LabelKeyConstant.java index bcb14a7045..8358e43ce9 100644 --- a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/constant/LabelKeyConstant.java +++ b/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/constant/LabelKeyConstant.java @@ -63,4 +63,8 @@ public class LabelKeyConstant { public static final String TENANT_KEY = "tenant"; public static final String FIXED_EC_KEY = "fixedEngineConn"; + + public static final String TEMPLATE_CONF_KEY = "ec.conf.templateId"; + + public static final String MANAGER_KEY = "manager"; } diff --git a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/cluster/ClusterLabel.java b/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/cluster/ClusterLabel.java index c0fa569594..5e1a3d5c23 100644 --- a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/cluster/ClusterLabel.java +++ b/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/cluster/ClusterLabel.java @@ -18,13 +18,18 @@ package org.apache.linkis.manager.label.entity.cluster; import org.apache.linkis.manager.label.constant.LabelKeyConstant; -import org.apache.linkis.manager.label.entity.Feature; -import org.apache.linkis.manager.label.entity.GenericLabel; +import org.apache.linkis.manager.label.entity.*; import org.apache.linkis.manager.label.entity.annon.ValueSerialNum; +import org.apache.linkis.manager.label.exception.LabelErrorException; + +import org.apache.commons.lang3.StringUtils; import java.util.HashMap; -public class ClusterLabel extends GenericLabel { +import static org.apache.linkis.manager.label.errorcode.LabelCommonErrorCodeSummary.CLUSTER_LABEL_ERROR_CODE; + +public class ClusterLabel extends GenericLabel + implements EMNodeLabel, EngineNodeLabel, UserModifiable { public ClusterLabel() { setLabelKey(LabelKeyConstant.YARN_CLUSTER_KEY); @@ -64,4 +69,14 @@ public String getClusterType() { } return null; } + + @Override + public void valueCheck(String stringValue) throws LabelErrorException { + if (!StringUtils.isEmpty(stringValue)) { + if (stringValue.split(SerializableLabel.VALUE_SEPARATOR).length != 2) { + throw new LabelErrorException( + CLUSTER_LABEL_ERROR_CODE.getErrorCode(), CLUSTER_LABEL_ERROR_CODE.getErrorDesc()); + } + } + } } diff --git a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/engine/ManagerLabel.java b/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/engine/ManagerLabel.java new file mode 100644 index 0000000000..674cc605af --- /dev/null +++ b/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/engine/ManagerLabel.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.label.entity.engine; + +import org.apache.linkis.manager.label.constant.LabelKeyConstant; +import org.apache.linkis.manager.label.entity.EngineNodeLabel; +import org.apache.linkis.manager.label.entity.Feature; +import org.apache.linkis.manager.label.entity.GenericLabel; +import org.apache.linkis.manager.label.entity.annon.ValueSerialNum; + +import java.util.HashMap; + +public class ManagerLabel extends GenericLabel implements EngineNodeLabel { + + public ManagerLabel() { + setLabelKey(LabelKeyConstant.MANAGER_KEY); + } + + @Override + public Feature getFeature() { + return Feature.CORE; + } + + public String getManager() { + if (null == getValue()) { + return null; + } + return getValue().get(getLabelKey()); + } + + @ValueSerialNum(0) + public void setManager(String manager) { + if (null == getValue()) { + setValue(new HashMap<>()); + } + getValue().put(getLabelKey(), manager); + } +} diff --git a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/engine/RunType.scala b/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/engine/RunType.scala index f8ba133f32..7ef0ce0a5e 100644 --- a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/engine/RunType.scala +++ b/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/engine/RunType.scala @@ -42,6 +42,7 @@ object RunType extends Enumeration { val ES_JSON = Value("esjson") val TRINO_SQL = Value("tsql") + val JSON = Value("json") val SEATUNNEL_FLINK_SQL = Value("sfsql") diff --git a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/errorcode/LabelCommonErrorCodeSummary.java b/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/errorcode/LabelCommonErrorCodeSummary.java index a0ebdffe8a..7ce6c6546d 100644 --- a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/errorcode/LabelCommonErrorCodeSummary.java +++ b/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/errorcode/LabelCommonErrorCodeSummary.java @@ -21,6 +21,9 @@ public enum LabelCommonErrorCodeSummary implements LinkisErrorCode { UPDATE_LABEL_FAILED(25001, "Update label realtion failed(更新标签属性失败)"), + CLUSTER_LABEL_ERROR_CODE( + 25002, + "The value of the label is set incorrectly, the setting value is: ClusterType-ClusterName "), LABEL_ERROR_CODE( 25002, "The value of the label is set incorrectly, only one value can be set, and the separator symbol '-' cannot be used(标签的值设置错误,只能设置一个值，不能使用分割符符号 '-') "), diff --git a/linkis-computation-governance/linkis-manager/linkis-label-common/src/test/java/org/apache/linkis/manager/label/builder/factory/StdLabelBuilderFactoryTest.java b/linkis-computation-governance/linkis-manager/linkis-label-common/src/test/java/org/apache/linkis/manager/label/builder/factory/StdLabelBuilderFactoryTest.java index 8d0b1e4de9..50f4cb44cb 100644 --- a/linkis-computation-governance/linkis-manager/linkis-label-common/src/test/java/org/apache/linkis/manager/label/builder/factory/StdLabelBuilderFactoryTest.java +++ b/linkis-computation-governance/linkis-manager/linkis-label-common/src/test/java/org/apache/linkis/manager/label/builder/factory/StdLabelBuilderFactoryTest.java @@ -19,11 +19,16 @@ import org.apache.linkis.manager.label.builder.CombinedLabelBuilder; import org.apache.linkis.manager.label.builder.LabelBuilder; +import org.apache.linkis.manager.label.entity.Label; import org.apache.linkis.manager.label.entity.em.EMInstanceLabel; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + import org.junit.jupiter.api.*; /** StdLabelBuilderFactory Tester */ @@ -73,4 +78,14 @@ public void testCreateLabelForInLabelKeyInValueStreamOutLabelClassOutValueTypes( stdLabelBuilderFactory.createLabel("testLabelKey", null, EMInstanceLabel.class, null); Assertions.assertTrue(emInstanceLabel1.getLabelKey().equals("emInstance")); } + + @Test + public void test() { + Map input = new HashMap(); + input.put("userCreator", "username-IDE"); + input.put("yarnCluster", "bdp-test"); + input.put("executeOnce", "true"); + List